Agenty

Hosted Web Scraping Tool

The world’s best companies use Agenty to bring web data to their business.

Try it free for 14 days
Watch the Video
Scraper

Advanced Web Data Extraction Made Easy

Everything you need to automate your data scraping

Start in Minutes

Create a web scraping agent with just a few clicks and start extracting data online

Built to Scale

We're already crawling billions of pages from thousands of website every month

Post Data to Server

Use the webhook trigger to post data on your server or upload to secure FTP

Alerts and Monitoring

Get automatic email alert when your web scraping job completes

Historical Data

Re-use all the data you've scraped ever for your analysis

Scheduling

Schedule your scraping agent to extract data when you are offline

Crawling Logs

View the details of all your runs and any failed URL with event and messages

Distributed Web Crawling

Automatic distribution of scraping agent over multiple machines to boost the performance

Advance Scripting

Write your own script to modify the scraped data into your choice of format

Collect data from every source on the web.

Power your business with the data it need.

Static and AJAX websites

List pages and search results

Password protected websites

Form submission websites

JSON and XML web APIs

XML and RSS sitemaps

Built by developers for developers

Forget the complexity of web: our hosted API, point-and-click Chrome app allow you to build in no time the best web scraping experience ever.

With C#(RestSharp) (Microsoft .NET)

1. Start the scraping agent 


var client = new RestClient("https://api.agenty.com/v1/start/8be99078d0");
var request = new RestRequest(Method.POST);
request.AddHeader("content-type", "application/json");
request.AddHeader("X-DataScraping-Api-Key", "*********************");
request.AddHeader("X-DataScraping-Api-Id", "*************");
IRestResponse response = client.Execute(request);

2. Get the data 


var client = new RestClient("https://api.agenty.com/v1/output/8be99078d0?offset=0&limit=1000");
var request = new RestRequest(Method.GET);
request.AddHeader("content-type", "application/json");
request.AddHeader("X-DataScraping-Api-Key", "*********************");
request.AddHeader("X-DataScraping-Api-Id", "*************");
IRestResponse response = client.Execute(request);

With cURL

1. Start the scraping agent 


curl -X POST -H "X-DataScraping-Api-Id: *************" -H "X-DataScraping-Api-Key: ******************************" -H "Content-Type: application/json" -d '' "https://api.agenty.com/v1/start/8be99078d0"

2. Get the data 


curl -X GET -H "X-DataScraping-API-Id: *************" -H "X-DataScraping-API-Key: ******************************" -H "Content-Type: application/json" "https://api.agenty.com/v1/output/8be99078d0?offset=0&limit=1000"

With Go

1. Start the scraping agent 


package main

import (
	"fmt"
	"net/http"
	"io/ioutil"
)

func main() {

	url := "https://api.agenty.com/v1/start/8be99078d0"

	req, _ := http.NewRequest("POST", url, nil)

	req.Header.Add("X-DataScraping-Api-Id", "*************")
	req.Header.Add("X-DataScraping-Api-Key", "******************************")
	req.Header.Add("content-type", "application/json")

	res, _ := http.DefaultClient.Do(req)

	defer res.Body.Close()
	body, _ := ioutil.ReadAll(res.Body)

	fmt.Println(res)
	fmt.Println(string(body))

}

2. Get the data 


package main

import (
	"fmt"
	"net/http"
	"io/ioutil"
)

func main() {

	url := "https://api.agenty.com/v1/output/8be99078d0?offset=0&limit=1000"

	req, _ := http.NewRequest("GET", url, nil)

	req.Header.Add("x-datascraping-api-id", "*************")
	req.Header.Add("x-datascraping-api-key", "******************************")
	req.Header.Add("content-type", "application/json")

	res, _ := http.DefaultClient.Do(req)

	defer res.Body.Close()
	body, _ := ioutil.ReadAll(res.Body)

	fmt.Println(res)
	fmt.Println(string(body))

}

With Java(OK HTTP)

1. Start the scraping agent 


OkHttpClient client = new OkHttpClient();

Request request = new Request.Builder()
  .url("https://api.agenty.com/v1/start/8be99078d0")
  .post(null)
  .addHeader("X-DataScraping-Api-Id", "*************")
  .addHeader("X-DataScraping-Api-Key", "******************************")
  .addHeader("content-type", "application/json")
  .build();

Response response = client.newCall(request).execute();

2. Get the data 


OkHttpClient client = new OkHttpClient();

Request request = new Request.Builder()
  .url("https://api.agenty.com/v1/output/8be99078d0?offset=0&limit=1000")
  .get()
  .addHeader("x-datascraping-api-id", "*************")
  .addHeader("x-datascraping-api-key", "******************************")
  .addHeader("content-type", "application/json")
  .build();

Response response = client.newCall(request).execute();

With Jquery AJAX

1. Start the scraping agent 


var settings = {
  "async": true,
  "crossDomain": true,
  "url": "https://api.agenty.com/v1/start/8be99078d0",
  "method": "POST",
  "headers": {
    "X-DataScraping-Api-Id": "*************",
    "X-DataScraping-Api-Key": "******************************",
    "content-type": "application/json"
  },
  "processData": false,
  "data": ""
}

$.ajax(settings).done(function (response) {
  console.log(response);
});

2. Get the data 


  var settings = {
  "async": true,
  "crossDomain": true,
  "url": "https://api.agenty.com/v1/output/8be99078d0?offset=0&limit=1000",
  "method": "GET",
  "headers": {
    "x-datascraping-api-id": "*************",
    "x-datascraping-api-key": "******************************",
    "content-type": "application/json"
  },
  "processData": false,
  "data": ""
}

$.ajax(settings).done(function (response) {
  console.log(response);
});

With NodeJS

1. Start the scraping agent 


var request = require("request");

var options = { method: 'POST',
  url: 'https://api.agenty.com/v1/start/8be99078d0',
  headers: 
   { 'content-type': 'application/json',
     'X-DataScraping-Api-Key': '******************************',
     'X-DataScraping-Api-Id': '*************' } };

request(options, function (error, response, body) {
  if (error) throw new Error(error);

  console.log(body);
});

2. Get the data 


var request = require("request");

var options = { method: 'GET',
  url: 'https://api.agenty.com/v1/output/8be99078d0',
  qs: { offset: '0', limit: '1000' },
  headers: 
   { 'content-type': 'application/json',
     'x-datascraping-api-key': '******************************',
     'x-datascraping-api-id': '*************' } };

request(options, function (error, response, body) {
  if (error) throw new Error(error);

  console.log(body);
});

With PHP

1. Start the scraping agent 


setUrl('https://api.agenty.com/v1/start/8be99078d0');
$request->setMethod(HTTP_METH_POST);

$request->setHeaders(array(
  'content-type' => 'application/json',
  'X-DataScraping-Api-Key' => '******************************',
  'X-DataScraping-Api-Id' => '*************'
));

try {
  $response = $request->send();

  echo $response->getBody();
} catch (HttpException $ex) {
  echo $ex;
}

2. Get the data 


setUrl('https://api.agenty.com/v1/output/8be99078d0');
$request->setMethod(HTTP_METH_GET);

$request->setQueryData(array(
  'offset' => '0',
  'limit' => '1000'
));

$request->setHeaders(array(
  'content-type' => 'application/json',
  'x-datascraping-api-key' => '******************************',
  'x-datascraping-api-id' => '*************'
));

try {
  $response = $request->send();

  echo $response->getBody();
} catch (HttpException $ex) {
  echo $ex;
}

With Python

1. Start the scraping agent 


import http.client

conn = http.client.HTTPSConnection("api.agenty.com")

payload = ""

headers = {
    'X-DataScraping-Api-Id': "*************",
    'X-DataScraping-Api-Key': "******************************",
    'content-type': "application/json"
    }

conn.request("POST", "/v1/start/8be99078d0", payload, headers)

res = conn.getresponse()
data = res.read()

print(data.decode("utf-8"))

2. Get the data 


import http.client

conn = http.client.HTTPSConnection("api.agenty.com")

payload = ""

headers = {
    'x-datascraping-api-id': "*************",
    'x-datascraping-api-key': "******************************",
    'content-type': "application/json"
    }

conn.request("GET", "/v1/output/8be99078d0?offset=0&limit=1000", payload, headers)

res = conn.getresponse()
data = res.read()

print(data.decode("utf-8"))

With Ruby

1. Start the scraping agent 


require 'uri'
require 'net/http'

url = URI("https://api.agenty.com/v1/start/8be99078d0")

http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
http.verify_mode = OpenSSL::SSL::VERIFY_NONE

request = Net::HTTP::Post.new(url)
request["X-DataScraping-Api-Id"] = '*************'
request["X-DataScraping-Api-Key"] = '******************************'
request["content-type"] = 'application/json'

response = http.request(request)
puts response.read_body

2. Get the data 


require 'uri'
require 'net/http'

url = URI("https://api.agenty.com/v1/output/8be99078d0?offset=0&limit=1000")

http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
http.verify_mode = OpenSSL::SSL::VERIFY_NONE

request = Net::HTTP::Get.new(url)
request["x-datascraping-api-id"] = '*************'
request["x-datascraping-api-key"] = '******************************'
request["content-type"] = 'application/json'

response = http.request(request)
puts response.read_body
API Documentation

Sign up to get free API key.

Try it free with 1000 pages credit

  • Easy setup
  • Super-fast crawling
  • API ready
  • Cancel at any time

Join thousands of businesses who use our scraping app to bring web data to their business

Used by 1000 of users

Data As A {service}

You need data? Our data expert team can help!

From small one-time project to high volume daily/weekly or monthly data feeds, we have the solution and experience to deliver. Let the expert data scraping team build, maintain, and host your data scraping project. No implementation, no learning curve, no hassle. Just the data you need, the way you need it.

Contact sales

Try free demo. No contract, commitment required!

Learn more about our managed web scraping service.

Close me