Documentation
¶
Overview ¶
Package acurl is the Go API behind the acurl command: it fetches a URL once per agent profile and reports how each agent was treated.
c, err := acurl.New(acurl.Options{})
if err != nil { ... }
rep, err := c.Check(ctx, "https://shop.example/products/1")
fmt.Println(rep.Score.Value)
acurl is an open-source project by tragent.io (https://tragent.io).
Index ¶
Examples ¶
Constants ¶
This section is empty.
Variables ¶
This section is empty.
Functions ¶
Types ¶
type Checker ¶
type Checker struct {
// contains filtered or unexported fields
}
Checker runs checks. It is safe for concurrent use and shares its HTTP connections and robots.txt cache across URLs.
func (*Checker) Check ¶
Check fetches rawURL with every profile and returns the classified, scored report. Per-profile network failures are reported in the results; the error is only for invalid input.
Example ¶
package main
import (
"context"
"fmt"
"net/http"
"net/http/httptest"
"strings"
"github.com/tragent-io/acurl/pkg/acurl"
"github.com/tragent-io/acurl/pkg/types"
)
func store() *httptest.Server {
return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path == "/robots.txt" {
_, _ = w.Write([]byte("User-agent: GPTBot\nDisallow: /\n"))
return
}
if strings.Contains(r.UserAgent(), "PerplexityBot") {
w.WriteHeader(http.StatusForbidden)
return
}
_, _ = w.Write([]byte(`<html><head><title>Runner</title>
<script type="application/ld+json">{"@type":"Product","name":"Runner","offers":{"price":"98","priceCurrency":"USD","availability":"https://schema.org/InStock"}}</script>
</head><body><main><h1>Runner</h1><p>Merino sneakers with free shipping and returns, in sizes 8 to 14.</p></main></body></html>`))
}))
}
func profiles(names ...string) []types.Profile {
all := map[string]types.Profile{
"Chrome": {Name: "Chrome", UserAgent: "Mozilla/5.0 Chrome/120", Browser: true, Category: types.CategoryBrowser},
"ChatGPT-User": {Name: "ChatGPT-User", UserAgent: "ChatGPT-User/1.0", RobotsKey: "ChatGPT-User", Category: types.CategoryAIAssistant},
"PerplexityBot": {Name: "PerplexityBot", UserAgent: "PerplexityBot/1.0", RobotsKey: "PerplexityBot", Category: types.CategoryAISearch},
"GPTBot": {Name: "GPTBot", UserAgent: "GPTBot/1.1", RobotsKey: "GPTBot", Category: types.CategoryAITraining},
}
var out []types.Profile
for _, n := range names {
out = append(out, all[n])
}
return out
}
func main() {
srv := store()
defer srv.Close()
c, _ := acurl.New(acurl.Options{Profiles: profiles("Chrome", "ChatGPT-User")})
rep, _ := c.Check(context.Background(), srv.URL+"/products/runner")
for _, r := range rep.Results {
fmt.Println(r.Profile, r.Classification)
}
}
Output: Chrome allowed ChatGPT-User allowed
type Options ¶
type Options struct {
// Profiles to fetch with. Nil means the built-in catalog.
Profiles []types.Profile
// Concurrency is the number of parallel requests per URL. Default 5.
Concurrency int
Timeout time.Duration // per request, default 30s
Headers map[string]string // extra headers for every request
Retries int // retries after network or gateway errors
Proxy string // http, https or socks5 proxy URL
Insecure bool // skip TLS verification
HTTPVersion string // "", "1.1" or "2"
Method string // default GET
BotTLS string // "go" (default) or "chrome"
MaxBody int64 // body cap per response, default 5 MiB
// SkipRobots disables robots.txt checks and drops robots-only profiles.
SkipRobots bool
// CloakThreshold is the similarity below which a response is cloaked.
// Zero means the default (0.5); a negative value disables detection.
CloakThreshold float64
// Render also renders the page in headless Chrome (must be installed).
Render bool
RenderWait time.Duration // default 2s
// Samples checks the page this many times and reports each agent's most
// common verdict, marking agents whose verdict changed as flaky.
// Default 1. SampleDelay is the pause between samples, default 1s.
Samples int
SampleDelay time.Duration
// Page says what the page is: "product", "other", or "" to decide from
// the URL and the page. Product pages are expected to have product data.
Page string
// UserAgent identifies acurl when it fetches robots.txt and llms.txt.
UserAgent string
// Trace, if set, is called after every page request.
Trace func(TraceEvent)
// SigningKey, if set, signs every bot-profile request with Web Bot Auth
// (HTTP Message Signatures, RFC 9421). SignatureAgent is the URL where
// the key directory is published.
SigningKey ed25519.PrivateKey
SignatureAgent string
// BlockPrivate refuses to connect to private, loopback, link-local and
// cloud-metadata addresses. Enable it when URLs come from untrusted input.
BlockPrivate bool
}
Options configures a Checker. The zero value checks with the default profiles and sensible network settings.
type TraceEvent ¶
type TraceEvent = fetcher.TraceEvent
TraceEvent describes one completed page request.
Source Files
Click to show internal directories.
Click to hide internal directories.