Add Go-based PokeAPI fetch tool
Replaces the Python fetch_pokeapi.py script with a Go tool that crawls a local PokeAPI instance and writes seed JSON files. Supports caching and special encounter definitions via JSON config. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
106
tools/fetch-pokeapi/client.go
Normal file
106
tools/fetch-pokeapi/client.go
Normal file
@@ -0,0 +1,106 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Client is an HTTP client for the PokeAPI with disk caching and concurrency limiting.
|
||||
type Client struct {
|
||||
baseURL string
|
||||
httpClient *http.Client
|
||||
cacheDir string
|
||||
sem chan struct{} // concurrency limiter
|
||||
}
|
||||
|
||||
// NewClient creates a new PokeAPI client.
|
||||
func NewClient(baseURL, cacheDir string, concurrency int) *Client {
|
||||
return &Client{
|
||||
baseURL: strings.TrimRight(baseURL, "/"),
|
||||
httpClient: &http.Client{
|
||||
Timeout: 2 * time.Minute,
|
||||
},
|
||||
cacheDir: cacheDir,
|
||||
sem: make(chan struct{}, concurrency),
|
||||
}
|
||||
}
|
||||
|
||||
// Get fetches the given endpoint, using disk cache when available.
|
||||
func (c *Client) Get(ctx context.Context, endpoint string) ([]byte, error) {
|
||||
// Check cache first (no semaphore needed for disk reads)
|
||||
safeName := strings.NewReplacer("/", "_", "?", "_").Replace(endpoint) + ".json"
|
||||
cachePath := filepath.Join(c.cacheDir, safeName)
|
||||
|
||||
if data, err := os.ReadFile(cachePath); err == nil {
|
||||
return data, nil
|
||||
}
|
||||
|
||||
// Acquire semaphore for HTTP request
|
||||
select {
|
||||
case c.sem <- struct{}{}:
|
||||
defer func() { <-c.sem }()
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
|
||||
url := c.baseURL + "/" + endpoint
|
||||
|
||||
var data []byte
|
||||
maxRetries := 3
|
||||
for attempt := range maxRetries {
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("creating request for %s: %w", endpoint, err)
|
||||
}
|
||||
|
||||
resp, err := c.httpClient.Do(req)
|
||||
if err != nil {
|
||||
if attempt < maxRetries-1 {
|
||||
backoff := time.Duration(1<<uint(attempt)) * time.Second
|
||||
time.Sleep(backoff)
|
||||
continue
|
||||
}
|
||||
return nil, fmt.Errorf("fetching %s: %w", endpoint, err)
|
||||
}
|
||||
|
||||
body, readErr := io.ReadAll(resp.Body)
|
||||
resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
if attempt < maxRetries-1 && resp.StatusCode >= 500 {
|
||||
backoff := time.Duration(1<<uint(attempt)) * time.Second
|
||||
time.Sleep(backoff)
|
||||
continue
|
||||
}
|
||||
return nil, fmt.Errorf("fetching %s: status %d", endpoint, resp.StatusCode)
|
||||
}
|
||||
|
||||
if readErr != nil {
|
||||
return nil, fmt.Errorf("reading response for %s: %w", endpoint, readErr)
|
||||
}
|
||||
|
||||
data = body
|
||||
break
|
||||
}
|
||||
|
||||
// Write to cache
|
||||
if err := os.MkdirAll(c.cacheDir, 0o755); err != nil {
|
||||
return nil, fmt.Errorf("creating cache dir: %w", err)
|
||||
}
|
||||
if err := os.WriteFile(cachePath, data, 0o644); err != nil {
|
||||
return nil, fmt.Errorf("writing cache for %s: %w", endpoint, err)
|
||||
}
|
||||
|
||||
return data, nil
|
||||
}
|
||||
|
||||
// ClearCache removes the cache directory.
|
||||
func (c *Client) ClearCache() error {
|
||||
return os.RemoveAll(c.cacheDir)
|
||||
}
|
||||
Reference in New Issue
Block a user