Documentation
¶
Overview ¶
Package batchclient provides high-performance batch execution capabilities for HTTP requests. It implements the httpclient.Driver interface and delegates requests to underlying driver implementations. This package is designed to work with any httpclient.Driver implementation and provides: - Concurrent batch execution of multiple requests - Rate limiting capabilities - Connection pooling management - Performance monitoring and statistics
Example (ChainedExecutors) ¶
package main
import (
"context"
"fmt"
"log"
"time"
"github.com/geniusrabbit/adcorelib/net/httpclient"
"github.com/geniusrabbit/adcorelib/net/httpclient/batchclient"
"github.com/geniusrabbit/adcorelib/net/httpclient/stdhttpclient"
)
func main() {
// Create a base HTTP client driver
driver := stdhttpclient.NewDriver()
// Chain multiple executors together
// First: Connection pool management
pool := batchclient.NewConnectionPool(driver)
pool.SetRetryPolicy(2, 50*time.Millisecond)
// Second: Rate limiting
rateLimiter := batchclient.NewRateLimitedExecutor(pool, 20, 10)
// Third: Batch execution
batchExecutor := batchclient.NewBatchExecutor(rateLimiter, 50)
// Create requests
var requests []httpclient.Request
for i := 0; i < 10; i++ {
req, err := batchExecutor.Request("GET", "https://httpbin.org/get", nil)
if err != nil {
log.Printf("Failed to create request %d: %v", i, err)
continue
}
requests = append(requests, req)
}
// Execute with all features: connection pooling, rate limiting, and batch processing
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
fmt.Println("Executing with chained executors (pool + rate limiter + batch)...")
start := time.Now()
result := batchExecutor.ExecuteBatch(ctx, requests)
elapsed := time.Since(start)
fmt.Printf("Chained execution completed in %v\n", elapsed)
fmt.Printf("Completed: %d/%d requests\n", result.Completed, len(result.Requests))
// Clean up responses
for _, req := range result.Requests {
if req.Response != nil {
_ = req.Response.Close()
}
}
// Print all statistics
fmt.Println("\nStatistics:")
batchStats := batchExecutor.Stats()
fmt.Printf(" Batch - Total: %d, Completed: %d, Errors: %d\n",
batchStats.TotalRequests, batchStats.TotalCompleted, batchStats.TotalErrors)
rateStats := rateLimiter.Stats()
fmt.Printf(" Rate Limiter - Total: %d, Executed: %d, Rate: %d/s\n",
rateStats.TotalRequests, rateStats.TotalExecuted, rateStats.RequestsPerSecond)
poolStats := pool.Stats()
fmt.Printf(" Pool - Connections: %d, Attempts: %d, Success Rate: %.2f%%\n",
poolStats.TotalConnections, poolStats.ConnectionAttempts, poolStats.SuccessRate*100)
// Clean up
rateLimiter.Stop()
pool.Stop()
}
Output:
Example (WithCustomDriver) ¶
Example of using batchclient with a custom driver
package main
import (
"context"
"fmt"
"io"
"log"
"github.com/geniusrabbit/adcorelib/net/httpclient"
"github.com/geniusrabbit/adcorelib/net/httpclient/batchclient"
"github.com/geniusrabbit/adcorelib/net/httpclient/stdhttpclient"
)
func main() {
// Create a custom driver that logs requests
customDriver := &loggingDriver{
underlying: stdhttpclient.NewDriver(),
}
// Use batch execution with the custom driver
batchExecutor := batchclient.NewBatchExecutor(customDriver, 10)
// Create and execute requests
var requests []httpclient.Request
for i := 0; i < 3; i++ {
req, err := batchExecutor.Request("GET", "https://httpbin.org/get", nil)
if err != nil {
log.Printf("Failed to create request %d: %v", i, err)
continue
}
requests = append(requests, req)
}
ctx := context.Background()
result := batchExecutor.ExecuteBatch(ctx, requests)
fmt.Printf("Custom driver executed %d requests\n", result.Completed)
// Clean up
for _, req := range result.Requests {
if req.Response != nil {
_ = req.Response.Close()
}
}
}
// loggingDriver is a custom driver that logs all requests
type loggingDriver struct {
underlying httpclient.Driver
}
func (ld *loggingDriver) Request(method, url string, body io.Reader) (httpclient.Request, error) {
log.Printf("Creating request: %s %s", method, url)
return ld.underlying.Request(method, url, body)
}
func (ld *loggingDriver) Do(req httpclient.Request) (httpclient.Response, error) {
log.Printf("Executing request")
return ld.underlying.Do(req)
}
Output:
Index ¶
- type BatchExecutor
- func (be *BatchExecutor) Do(req httpclient.Request) (httpclient.Response, error)
- func (be *BatchExecutor) ExecuteBatch(ctx context.Context, requests []httpclient.Request) BatchResult
- func (be *BatchExecutor) Request(method, url string, body io.Reader) (httpclient.Request, error)
- func (be *BatchExecutor) ResetStats()
- func (be *BatchExecutor) Stats() BatchStats
- type BatchRequest
- type BatchResult
- type BatchStats
- type ConnectionPool
- func (cp *ConnectionPool) DisableHealthCheck()
- func (cp *ConnectionPool) Do(req httpclient.Request) (httpclient.Response, error)
- func (cp *ConnectionPool) EnableHealthCheck(hosts []string, interval time.Duration)
- func (cp *ConnectionPool) Request(method, url string, body io.Reader) (httpclient.Request, error)
- func (cp *ConnectionPool) ResetStats()
- func (cp *ConnectionPool) SetRetryPolicy(maxRetries int, retryDelay time.Duration)
- func (cp *ConnectionPool) Stats() ConnectionPoolStats
- func (cp *ConnectionPool) Stop()
- func (cp *ConnectionPool) WarmupConnections(ctx context.Context, hosts []string, connectionsPerHost int) error
- type ConnectionPoolStats
- type RateLimitedExecutor
- func (rle *RateLimitedExecutor) Do(req httpclient.Request) (httpclient.Response, error)
- func (rle *RateLimitedExecutor) Execute(ctx context.Context, req httpclient.Request) (httpclient.Response, error)
- func (rle *RateLimitedExecutor) ExecuteWithTimeout(req httpclient.Request, timeout time.Duration) (httpclient.Response, error)
- func (rle *RateLimitedExecutor) Request(method, url string, body io.Reader) (httpclient.Request, error)
- func (rle *RateLimitedExecutor) ResetStats()
- func (rle *RateLimitedExecutor) Stats() RateLimiterStats
- func (rle *RateLimitedExecutor) Stop()
- func (rle *RateLimitedExecutor) UpdateRate(requestsPerSecond int)
- type RateLimiterStats
Examples ¶
Constants ¶
This section is empty.
Variables ¶
This section is empty.
Functions ¶
This section is empty.
Types ¶
type BatchExecutor ¶
type BatchExecutor struct {
// contains filtered or unexported fields
}
BatchExecutor provides high-performance batch execution of HTTP requests. It implements httpclient.Driver interface and can be used as a drop-in replacement for any driver while providing batch processing capabilities.
Example ¶
package main
import (
"context"
"fmt"
"log"
"time"
"github.com/geniusrabbit/adcorelib/net/httpclient"
"github.com/geniusrabbit/adcorelib/net/httpclient/batchclient"
"github.com/geniusrabbit/adcorelib/net/httpclient/stdhttpclient"
)
func main() {
// Create a base HTTP client driver
driver := stdhttpclient.NewDriver()
// Wrap it with batch execution capabilities
batchExecutor := batchclient.NewBatchExecutor(driver, 100) // 100 concurrent workers
// Create multiple requests
var requests []httpclient.Request
for i := 0; i < 10; i++ {
req, err := batchExecutor.Request("GET", "https://httpbin.org/delay/1", nil)
if err != nil {
log.Printf("Failed to create request %d: %v", i, err)
continue
}
requests = append(requests, req)
}
// Execute all requests concurrently
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
start := time.Now()
result := batchExecutor.ExecuteBatch(ctx, requests)
elapsed := time.Since(start)
// Print results
fmt.Printf("Batch execution completed in %v\n", elapsed)
fmt.Printf("Total requests: %d\n", len(result.Requests))
fmt.Printf("Completed: %d\n", result.Completed)
fmt.Printf("Errors: %d\n", result.Errors)
// Clean up responses
for _, req := range result.Requests {
if req.Response != nil {
_ = req.Response.Close()
}
}
// Print statistics
stats := batchExecutor.Stats()
fmt.Printf("Statistics - Total: %d, Completed: %d, Errors: %d\n",
stats.TotalRequests, stats.TotalCompleted, stats.TotalErrors)
}
Output:
func NewBatchExecutor ¶
func NewBatchExecutor(driver httpclient.Driver, maxWorkers int) *BatchExecutor
NewBatchExecutor creates a new batch executor with the specified driver and worker count. The driver parameter can be any implementation of httpclient.Driver interface. If maxWorkers is <= 0, it defaults to 100 workers.
func (*BatchExecutor) Do ¶
func (be *BatchExecutor) Do(req httpclient.Request) (httpclient.Response, error)
Do implements httpclient.Driver interface. Executes a single HTTP request using the underlying driver.
func (*BatchExecutor) ExecuteBatch ¶
func (be *BatchExecutor) ExecuteBatch(ctx context.Context, requests []httpclient.Request) BatchResult
ExecuteBatch executes multiple HTTP requests concurrently with optimal performance. This is the main batch processing method that provides high-throughput execution.
func (*BatchExecutor) Request ¶
func (be *BatchExecutor) Request(method, url string, body io.Reader) (httpclient.Request, error)
Request implements httpclient.Driver interface. Creates a new HTTP request using the underlying driver.
func (*BatchExecutor) ResetStats ¶
func (be *BatchExecutor) ResetStats()
ResetStats resets all execution statistics.
func (*BatchExecutor) Stats ¶
func (be *BatchExecutor) Stats() BatchStats
Stats returns execution statistics.
type BatchRequest ¶
type BatchRequest struct {
Request httpclient.Request
Response httpclient.Response
Error error
Index int // Original index in the batch
}
BatchRequest represents a single request in a batch operation.
type BatchResult ¶
type BatchResult struct {
Requests []BatchRequest
Completed int
Errors int
Duration time.Duration
StartTime time.Time
EndTime time.Time
}
BatchResult contains the results of a batch operation.
type BatchStats ¶
type BatchStats struct {
TotalRequests int64
TotalCompleted int64
TotalErrors int64
TotalDuration time.Duration
AverageDuration time.Duration
ErrorRate float64
}
BatchStats contains execution statistics.
type ConnectionPool ¶
type ConnectionPool struct {
// contains filtered or unexported fields
}
ConnectionPool provides connection pooling statistics and management. It wraps any httpclient.Driver implementation and provides connection management features.
Example ¶
package main
import (
"context"
"fmt"
"log"
"time"
"github.com/geniusrabbit/adcorelib/net/httpclient/batchclient"
"github.com/geniusrabbit/adcorelib/net/httpclient/stdhttpclient"
)
func main() {
// Create a base HTTP client driver
driver := stdhttpclient.NewDriver()
// Wrap it with connection pool management
pool := batchclient.NewConnectionPool(driver)
// Configure retry policy
pool.SetRetryPolicy(3, 100*time.Millisecond)
// Warmup connections to target hosts
hosts := []string{
"https://httpbin.org",
"https://jsonplaceholder.typicode.com",
}
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
fmt.Println("Warming up connections...")
err := pool.WarmupConnections(ctx, hosts, 2)
if err != nil {
log.Printf("Warmup error: %v", err)
}
// Enable health checking
pool.EnableHealthCheck(hosts, 30*time.Second)
// Execute requests with connection pooling
for i := 0; i < 3; i++ {
req, err := pool.Request("GET", "https://httpbin.org/get", nil)
if err != nil {
log.Printf("Failed to create request %d: %v", i, err)
continue
}
resp, err := pool.Do(req)
if err != nil {
log.Printf("Failed to execute request %d: %v", i, err)
continue
}
fmt.Printf("Request %d completed with status %d\n", i+1, resp.StatusCode())
_ = resp.Close()
}
// Print statistics
stats := pool.Stats()
fmt.Printf("Pool stats - Connections: %d, Attempts: %d, Failures: %d, Success Rate: %.2f%%\n",
stats.TotalConnections, stats.ConnectionAttempts, stats.ConnectionFailures, stats.SuccessRate*100)
// Clean up
pool.Stop()
}
Output:
func NewConnectionPool ¶
func NewConnectionPool(driver httpclient.Driver) *ConnectionPool
NewConnectionPool creates a new connection pool manager. The driver parameter can be any implementation of httpclient.Driver interface.
func (*ConnectionPool) DisableHealthCheck ¶
func (cp *ConnectionPool) DisableHealthCheck()
DisableHealthCheck disables periodic health checking.
func (*ConnectionPool) Do ¶
func (cp *ConnectionPool) Do(req httpclient.Request) (httpclient.Response, error)
Do implements httpclient.Driver interface. Executes a HTTP request with connection pooling and retry logic.
func (*ConnectionPool) EnableHealthCheck ¶
func (cp *ConnectionPool) EnableHealthCheck(hosts []string, interval time.Duration)
EnableHealthCheck enables periodic health checking for the specified hosts.
func (*ConnectionPool) Request ¶
func (cp *ConnectionPool) Request(method, url string, body io.Reader) (httpclient.Request, error)
Request implements httpclient.Driver interface. Creates a new HTTP request using the underlying driver.
func (*ConnectionPool) ResetStats ¶
func (cp *ConnectionPool) ResetStats()
ResetStats resets all connection pool statistics.
func (*ConnectionPool) SetRetryPolicy ¶
func (cp *ConnectionPool) SetRetryPolicy(maxRetries int, retryDelay time.Duration)
SetRetryPolicy configures the retry policy for failed connections.
func (*ConnectionPool) Stats ¶
func (cp *ConnectionPool) Stats() ConnectionPoolStats
Stats returns connection pool statistics.
func (*ConnectionPool) Stop ¶
func (cp *ConnectionPool) Stop()
Stop stops all background operations.
func (*ConnectionPool) WarmupConnections ¶
func (cp *ConnectionPool) WarmupConnections(ctx context.Context, hosts []string, connectionsPerHost int) error
WarmupConnections pre-establishes connections to the specified hosts. This can improve performance by avoiding connection overhead during actual requests.
type ConnectionPoolStats ¶
type ConnectionPoolStats struct {
TotalConnections int64
ActiveConnections int64
ConnectionAttempts int64
ConnectionFailures int64
SuccessRate float64
MaxRetries int
RetryDelay time.Duration
HealthCheckEnabled bool
HealthCheckHosts int
HealthCheckInterval time.Duration
}
ConnectionPoolStats contains connection pool statistics.
type RateLimitedExecutor ¶
type RateLimitedExecutor struct {
// contains filtered or unexported fields
}
RateLimitedExecutor provides rate-limited execution of HTTP requests. It implements httpclient.Driver interface and can wrap any underlying driver to provide rate limiting capabilities.
Example ¶
package main
import (
"context"
"fmt"
"log"
"time"
"github.com/geniusrabbit/adcorelib/net/httpclient/batchclient"
"github.com/geniusrabbit/adcorelib/net/httpclient/stdhttpclient"
)
func main() {
// Create a base HTTP client driver
driver := stdhttpclient.NewDriver()
// Wrap it with rate limiting (10 requests per second)
rateLimiter := batchclient.NewRateLimitedExecutor(driver, 10, 5)
// Execute requests - will be rate limited
ctx := context.Background()
start := time.Now()
for i := 0; i < 5; i++ {
req, err := rateLimiter.Request("GET", "https://httpbin.org/get", nil)
if err != nil {
log.Printf("Failed to create request %d: %v", i, err)
continue
}
resp, err := rateLimiter.Execute(ctx, req)
if err != nil {
log.Printf("Failed to execute request %d: %v", i, err)
continue
}
fmt.Printf("Request %d completed with status %d\n", i+1, resp.StatusCode())
_ = resp.Close()
}
elapsed := time.Since(start)
fmt.Printf("5 requests completed in %v (rate limited)\n", elapsed)
// Print statistics
stats := rateLimiter.Stats()
fmt.Printf("Rate limiter stats - Requests: %d, Executed: %d, Rate: %d/s\n",
stats.TotalRequests, stats.TotalExecuted, stats.RequestsPerSecond)
// Clean up
rateLimiter.Stop()
}
Output:
func NewRateLimitedExecutor ¶
func NewRateLimitedExecutor(driver httpclient.Driver, requestsPerSecond int, maxWorkers int) *RateLimitedExecutor
NewRateLimitedExecutor creates a new rate-limited executor. The driver parameter can be any implementation of httpclient.Driver interface. requestsPerSecond controls the maximum rate of request execution.
func (*RateLimitedExecutor) Do ¶
func (rle *RateLimitedExecutor) Do(req httpclient.Request) (httpclient.Response, error)
Do implements httpclient.Driver interface. Executes a HTTP request with rate limiting applied.
func (*RateLimitedExecutor) Execute ¶
func (rle *RateLimitedExecutor) Execute(ctx context.Context, req httpclient.Request) (httpclient.Response, error)
Execute executes a request with rate limiting. This method respects the configured rate limit and will block if necessary.
func (*RateLimitedExecutor) ExecuteWithTimeout ¶
func (rle *RateLimitedExecutor) ExecuteWithTimeout(req httpclient.Request, timeout time.Duration) (httpclient.Response, error)
ExecuteWithTimeout executes a request with both rate limiting and timeout.
func (*RateLimitedExecutor) Request ¶
func (rle *RateLimitedExecutor) Request(method, url string, body io.Reader) (httpclient.Request, error)
Request implements httpclient.Driver interface. Creates a new HTTP request using the underlying driver.
func (*RateLimitedExecutor) ResetStats ¶
func (rle *RateLimitedExecutor) ResetStats()
ResetStats resets all rate limiter statistics.
func (*RateLimitedExecutor) Stats ¶
func (rle *RateLimitedExecutor) Stats() RateLimiterStats
Stats returns rate limiter statistics.
func (*RateLimitedExecutor) Stop ¶
func (rle *RateLimitedExecutor) Stop()
Stop stops the rate limiter and all background goroutines.
func (*RateLimitedExecutor) UpdateRate ¶
func (rle *RateLimitedExecutor) UpdateRate(requestsPerSecond int)
UpdateRate changes the rate limit at runtime.