Skip to content

Find, verify, and analyze leaked credentials

11.7 KB
Raw Blame
package huggingface

import (
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"net/http"
	"net/url"
	"strings"
	"time"

	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
)

// Maps for API and HTML paths
var apiPaths = map[string]string{
	DATASET: DatasetsRoute,
	MODEL:   ModelsAPIRoute,
	SPACE:   SpacesRoute,
	BUCKET:  BucketsRoute,
}

var htmlPaths = map[string]string{
	DATASET: DatasetsRoute,
	MODEL:   "",
	SPACE:   SpacesRoute,
	BUCKET:  BucketsRoute,
}

type Author struct {
	Username string `json:"name"`
}

type Latest struct {
	Raw string `json:"raw"`
}

type Data struct {
	Latest Latest `json:"latest"`
}

type Event struct {
	Type      string `json:"type"`
	Author    Author `json:"author"`
	CreatedAt string `json:"createdAt"`
	Data      Data   `json:"data"`
	ID        string `json:"id"`
}

func (e Event) GetAuthor() string {
	return e.Author.Username
}

func (e Event) GetCreatedAt() string {
	return e.CreatedAt
}

func (e Event) GetID() string {
	return e.ID
}

type RepoData struct {
	FullName     string `json:"name"`
	ResourceType string `json:"type"`
}

type Discussion struct {
	ID        int      `json:"num"`
	IsPR      bool     `json:"isPullRequest"`
	CreatedAt string   `json:"createdAt"`
	Title     string   `json:"title"`
	Events    []Event  `json:"events"`
	Repo      RepoData `json:"repo"`
}

func (d Discussion) GetID() string {
	return fmt.Sprint(d.ID)
}

func (d Discussion) GetTitle() string {
	return d.Title
}

func (d Discussion) GetCreatedAt() string {
	return d.CreatedAt
}

func (d Discussion) GetRepo() string {
	return d.Repo.FullName
}

// GetDiscussionPath returns the path (ex: "/models/user/repo/discussions/1") for the discussion
func (d Discussion) GetDiscussionPath() string {
	basePath := fmt.Sprintf("%s/%s/%s", d.GetRepo(), DiscussionsRoute, d.GetID())
	if d.Repo.ResourceType == "model" {
		return basePath
	}
	return fmt.Sprintf("%s/%s", getResourceHTMLPath(d.Repo.ResourceType), basePath)
}

// GetGitPath returns the path (ex: "/models/user/repo.git") for the repo's git directory
func (d Discussion) GetGitPath() string {
	basePath := fmt.Sprintf("%s.git", d.GetRepo())
	if d.Repo.ResourceType == "model" {
		return basePath
	}
	return fmt.Sprintf("%s/%s", getResourceHTMLPath(d.Repo.ResourceType), basePath)
}

type DiscussionList struct {
	Discussions []Discussion `json:"discussions"`
}

type Repo struct {
	IsPrivate bool   `json:"private"`
	Owner     string `json:"author"`
	RepoID    string `json:"id"`
}

type Bucket struct {
	IsPrivate bool   `json:"private"`
	BucketID  string `json:"id"`
}

type BucketFile struct {
	Type string `json:"type"`
	Path string `json:"path"`
	Size int64  `json:"size"`
}

type HFClient struct {
	BaseURL    string
	APIKey     string
	HTTPClient *http.Client
	// downloadClient streams file bodies. Unlike HTTPClient it has no overall
	// Timeout, because http.Client.Timeout also interrupts Response.Body reads —
	// which would kill large/slow downloads handed to handlers.HandleFile.
	downloadClient *http.Client
}

// NewHFClient creates a new HF client
func NewHFClient(baseURL, apiKey string, timeout time.Duration) *HFClient {
	// Bound the connect + response-header phase (including redirect hops to the
	// CDN) without capping the body read; overall cancellation comes from the
	// request context. Built explicitly (rather than cloning DefaultTransport)
	// so it stays a *http.Transport even when tests swap the default transport.
	downloadTransport := &http.Transport{
		Proxy:                 http.ProxyFromEnvironment,
		ForceAttemptHTTP2:     true,
		MaxIdleConns:          100,
		IdleConnTimeout:       90 * time.Second,
		TLSHandshakeTimeout:   10 * time.Second,
		ExpectContinueTimeout: 1 * time.Second,
		ResponseHeaderTimeout: timeout,
	}

	return &HFClient{
		BaseURL: baseURL,
		APIKey:  apiKey,
		HTTPClient: &http.Client{
			Timeout: timeout,
		},
		downloadClient: &http.Client{
			Transport: downloadTransport,
		},
	}
}

// get makes a GET request to the Hugging Face API
// Note: not addressing rate limit, since it seems very permissive. (ex: "If \
// your account suddenly sends 10k requests then you’re likely to receive 503")
func (c *HFClient) get(ctx context.Context, url string, target interface{}) error {
	ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
	defer cancel()

	req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
	if err != nil {
		return fmt.Errorf("failed to create HuggingFace API request: %w", err)
	}

	req.Header.Set("Authorization", "Bearer "+c.APIKey)

	resp, err := c.HTTPClient.Do(req)
	if err != nil {
		return fmt.Errorf("failed to make request to HuggingFace API: %w", err)
	}

	if resp.StatusCode == http.StatusUnauthorized {
		return errors.New("invalid API key")
	}

	if resp.StatusCode == http.StatusForbidden {
		return errors.New("access to this repo is restricted and you are not in the authorized list. Visit the repository to ask for access")
	}

	defer func() { _ = resp.Body.Close() }()

	return json.NewDecoder(resp.Body).Decode(target)
}

// GetRepo retrieves repo from the Hugging Face API
func (c *HFClient) GetRepo(ctx context.Context, repoName string, resourceType string) (Repo, error) {
	var repo Repo
	url, err := buildAPIURL(c.BaseURL, resourceType, repoName)
	if err != nil {
		return repo, err
	}
	err = c.get(ctx, url, &repo)
	return repo, err
}

// ListDiscussions retrieves discussions from the Hugging Face API
func (c *HFClient) ListDiscussions(ctx context.Context, repoInfo repoInfo) (DiscussionList, error) {
	var discussions DiscussionList
	baseURL, err := buildAPIURL(c.BaseURL, string(repoInfo.resourceType), repoInfo.fullName)
	if err != nil {
		return discussions, err
	}
	url := fmt.Sprintf("%s/%s", baseURL, DiscussionsRoute)
	err = c.get(ctx, url, &discussions)
	return discussions, err
}

func (c *HFClient) GetDiscussionByID(ctx context.Context, repoInfo repoInfo, discussionID string) (Discussion, error) {
	var discussion Discussion
	baseURL, err := buildAPIURL(c.BaseURL, string(repoInfo.resourceType), repoInfo.fullName)
	if err != nil {
		return discussion, err
	}
	url := fmt.Sprintf("%s/%s/%s", baseURL, DiscussionsRoute, discussionID)
	err = c.get(ctx, url, &discussion)
	return discussion, err
}

// ListReposByAuthor retrieves repos from the Hugging Face API by author (user or org)
// Note: not addressing pagination b/c allow by default 1000 results, which should be enough for 99.99% of cases
func (c *HFClient) ListReposByAuthor(ctx context.Context, resourceType string, author string) ([]Repo, error) {
	var repos []Repo
	url := fmt.Sprintf("%s/%s/%s?limit=1000&author=%s", c.BaseURL, APIRoute, getResourceAPIPath(resourceType), author)
	err := c.get(ctx, url, &repos)
	return repos, err
}

// GetBucket retrieves bucket metadata from the Hugging Face API.
func (c *HFClient) GetBucket(ctx context.Context, bucketID string) (Bucket, error) {
	var bucket Bucket
	url, err := buildAPIURL(c.BaseURL, BUCKET, bucketID)
	if err != nil {
		return bucket, err
	}
	err = c.get(ctx, url, &bucket)
	return bucket, err
}

// ListBucketsByAuthor retrieves buckets from the Hugging Face API by author (user or org).
func (c *HFClient) ListBucketsByAuthor(ctx context.Context, author string) ([]Bucket, error) {
	var buckets []Bucket
	url := fmt.Sprintf("%s/%s/%s/%s?limit=%d", c.BaseURL, APIRoute, BucketsRoute, author, bucketsPagination)
	for url != "" {
		var page []Bucket
		next, err := c.getPage(ctx, url, &page)
		if err != nil {
			return buckets, err
		}
		buckets = append(buckets, page...)
		url = next
	}
	return buckets, nil
}

// ListBucketFiles recursively lists all files in a bucket, following Link
// header pagination.
func (c *HFClient) ListBucketFiles(ctx context.Context, bucketID string) ([]BucketFile, error) {
	var files []BucketFile
	url := fmt.Sprintf("%s/%s/%s/%s/tree?recursive=true&limit=%d", c.BaseURL, APIRoute, BucketsRoute, bucketID, bucketsPagination)
	for url != "" {
		var page []BucketFile
		next, err := c.getPage(ctx, url, &page)
		if err != nil {
			return files, err
		}
		files = append(files, page...)
		url = next
	}
	return files, nil
}

// DownloadBucketFile streams a file's content from a bucket via the resolve
// endpoint. The caller is responsible for closing the returned reader.
func (c *HFClient) DownloadBucketFile(ctx context.Context, bucketID string, path string) (io.ReadCloser, error) {
	// Escape each path segment individually so special characters (spaces,
	// '?', '#', etc.) are encoded while slashes are preserved as separators.
	downloadURL := fmt.Sprintf("%s/%s/%s/resolve/%s", c.BaseURL, BucketsRoute, bucketID, escapePathSegments(path))

	req, err := http.NewRequestWithContext(ctx, "GET", downloadURL, nil)
	if err != nil {
		return nil, fmt.Errorf("failed to create HuggingFace bucket download request: %w", err)
	}
	req.Header.Set("Authorization", "Bearer "+c.APIKey)

	resp, err := c.downloadClient.Do(req)
	if err != nil {
		return nil, fmt.Errorf("failed to download bucket file %s: %w", path, err)
	}
	if resp.StatusCode != http.StatusOK {
		_ = resp.Body.Close()
		return nil, fmt.Errorf("failed to download bucket file %s: status %d", path, resp.StatusCode)
	}
	return resp.Body, nil
}

// getPage makes a GET request like get, but additionally returns the "next"
// URL from the response Link header, if any, for paginated endpoints.
func (c *HFClient) getPage(ctx context.Context, url string, target interface{}) (string, error) {
	ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
	defer cancel()

	req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
	if err != nil {
		return "", fmt.Errorf("failed to create HuggingFace API request: %w", err)
	}
	req.Header.Set("Authorization", "Bearer "+c.APIKey)

	resp, err := c.HTTPClient.Do(req)
	if err != nil {
		return "", fmt.Errorf("failed to make request to HuggingFace API: %w", err)
	}
	defer func() { _ = resp.Body.Close() }()

	if resp.StatusCode == http.StatusUnauthorized {
		return "", errors.New("invalid API key")
	}
	if resp.StatusCode == http.StatusForbidden {
		return "", errors.New("access to this resource is restricted and you are not in the authorized list. Visit the resource to ask for access")
	}
	if resp.StatusCode != http.StatusOK {
		return "", fmt.Errorf("unexpected status %d from HuggingFace API", resp.StatusCode)
	}

	if err := json.NewDecoder(resp.Body).Decode(target); err != nil {
		return "", err
	}
	return parseNextLink(resp.Header.Get("Link")), nil
}

// escapePathSegments URL-encodes each slash-separated segment of a path while
// keeping the slashes as separators, e.g. "a b/c?d" -> "a%20b/c%3Fd".
func escapePathSegments(path string) string {
	segments := strings.Split(path, "/")
	for i, segment := range segments {
		segments[i] = url.PathEscape(segment)
	}
	return strings.Join(segments, "/")
}

// parseNextLink extracts the URL with rel="next" from a Link header value.
// Example: <https://huggingface.co/api/...?cursor=abc>; rel="next"
func parseNextLink(header string) string {
	for _, part := range strings.Split(header, ",") {
		sections := strings.Split(part, ";")
		if len(sections) < 2 {
			continue
		}
		urlPart := strings.Trim(strings.TrimSpace(sections[0]), "<>")
		for _, param := range sections[1:] {
			if strings.TrimSpace(param) == `rel="next"` {
				return urlPart
			}
		}
	}
	return ""
}

// getResourceAPIPath returns the API path for the given resource type
func getResourceAPIPath(resourceType string) string {
	return apiPaths[resourceType]
}

// getResourceHTMLPath returns the HTML path for the given resource type
func getResourceHTMLPath(resourceType string) string {
	return htmlPaths[resourceType]
}

func buildAPIURL(endpoint string, resourceType string, repoName string) (string, error) {
	if endpoint == "" || resourceType == "" || repoName == "" {
		return "", errors.New("endpoint, resourceType, and repoName must not be empty")
	}
	return fmt.Sprintf("%s/%s/%s/%s", endpoint, APIRoute, getResourceAPIPath(resourceType), repoName), nil
}