Git designed for AI agents.
package mvcc
import (
"bytes"
"context"
"crypto/rand"
"crypto/sha256"
"encoding/hex"
"errors"
"fmt"
"io"
"os"
"path/filepath"
"strings"
"sync"
"time"
"github.com/feder-cr/git-ai/internal/safe"
"github.com/feder-cr/git-ai/internal/structerr"
"golang.org/x/sync/errgroup"
)
const (
// latestManifestPointer is the string used in `prepare` to indicate
// that the latest manifest pointer should be fetched instead of a
// specific one. The value of the string does not matter, as long as it
// does not conflict with a real manifest hash.
latestManifestPointer = ""
)
// EnvVar is a helper struct to allow for better handling of the
// environment variables returned by the Cache.
type EnvVar struct {
Name string
Value string
}
// String returns the environment in the format "key=val"
func (e EnvVar) String() string {
return fmt.Sprintf("%s=%s", e.Name, e.Value)
}
// Cache handles all operations between an RPC and a repository's MVCC
// cache. A Cache instance should be created once per RPC, and it's lifetime
// should be tied to the lifetime of the RPC.
// It allows to warm a cache before Git commands can be executed on the
// repository, and to commit the changes at the end of the RPC.
type Cache interface {
// Prepare prepares the local MVCC cache so that Git operations can
// be performed on the repository. Under the hood, it should call
// object storage, fetch the manifest pointer, and make sure all
// required artifacts are in the mvcc cache.
// This is an async call, it should not wait for the cache to be ready
// before returning.
Prepare(ctx context.Context)
// PrepareWithManifest is used when the RPC already contains a manifest hash
// in the metadata. Example: request coming from Gitaly hooks.
// Instead of using the latestmanifest pointer, it will prepare the
// cache to use this manifest hash and use it for the duration
// of the RPC.
PrepareWithManifest(ctx context.Context, hash string)
// WaitUntilReady is a blocking call, and should unblock when the mvcc cache
// is ready. This is useful to start the process of updating the cache
// ahead of time, in parallel with other things, so that once the handler
// is ready to execute Git commands, the cache has already started warming up.
WaitUntilReady(ctx context.Context) error
// Commit commits the result of the RPC (only Mutator RPC).
Commit(ctx context.Context) error
// SetCommitConflictError registers the error Commit returns instead of
// ErrManifestPointerConflict when CAS fails on the manifest pointer.
SetCommitConflictError(err error)
// AfterCommit registers fn to run once Commit has published the changes
// of the RPC. It returns false, and fn never runs, when the cache is
// read-only or the RPC has already started committing or been cleaned up.
// fn does not run either when the RPC fails or Commit fails, as the
// changes of the RPC are then thrown away. It may be called concurrently
// with the RPC, for example from a hook request served while the RPC waits
// for Git.
AfterCommit(fn func()) bool
// Environment returns the list of environment variable to use
// when invoking Git.
// Ex:
// - `GIT_MVCC_MANIFEST`
// - `GIT_MVCC_MANIFEST_PATH`
Environment() []EnvVar
// BaseManifestPointer returns the base manifest pointer of the
// repository, the one resolved at the start of the RPC.
// WaitUntilReady must be called prior to calling this because
// it has to wait for the manifest to be fetched before it can
// return the value.
BaseManifestPointer() string
// ReadOnly returns true if the RPC is a read only RPC or false
// otherwise.
ReadOnly() bool
// WalkManifest visits every artifact of the manifest that this cache
// was prepared against.
WalkManifest(ctx context.Context, visitor ArtifactVisitor) error
}
// LocalCache is an implementation of Cache targeting the repository MVCC cache
// on the local filesystem.
//
// A cache belongs to one RPC and has no internal synchronization: Prepare, Commit and Cleanup
// must be called from a single goroutine, which is what the middleware does.
type LocalCache struct {
// rs is the ObjectStorage used to communicate with persistent storage
// to fetch the manifests and Git artifacts
rs ObjectStorage
// repositoryID is the repository ID for the repository related to the
// current instance of the MVCCCache. While this might be a relative
// path (i.e.: @hashed/ab/cd/abcd...), it is purposely called repository
// **ID** because it could theoretically be any kind of IDs. The concept
// of relative path is foreign to the cache implementation.
repositoryID string
// This the runtime directory of Gitaly. This is where sockets and other Gitaly related files
// are created during the lifetime of the Gitaly process. This is where we will also create the
// manifests file for writes RPC.
runtimeDir string
// storagePath is the path where to find the repository on disk
storagePath string
// environments holds the environment variables that must be passed down
// to the Git commands executing on this repository for the proper mvcc
// manifest top be used.
environments map[string]string
// readOnly tells if the RPC related to this cache instance is a read-only
// RPC or not.
readOnly bool
// prepareDone is used to synchronize between the prepare phase and the ready phase.
// This channel gets closed one cache warming is done.
prepareDone chan struct{}
// readyCh is closed once the preparation phase is done. It allows to wait for the
// prepare phase to finish, so that we can capture the error into prepareErr.
readyDone chan struct{}
// prepareErr is the error returned by `prepare()`. We cache this value so that
// subsequent call to `WaitForReady` get the same value everytime. This insures
// consistent behavior when called multiple time.
prepareErr error
// prepareOnce makes sure the cache is only warmed once.
prepareOnce sync.Once
// readyOnce insures that, if `prepare` returns an error, multiple calls to
// WaitForReady returns that same error. Unless of course the context is canceled.
// In that case it returns the context error.
readyOnce sync.Once
// base is the manifest pointer resolved at the start of a mutator RPC. It is
// the base against which the commit-time compare-and-swap is performed, and it
// carries the generation (when the backend tracks one) so Commit can advance
// without re-reading the head.
base ManifestPointer
// baseManifest is the parsed manifest referenced by base.Hash. It is used to
// compute which artifacts a mutator RPC newly produced.
baseManifest *Manifest
// committed guards Commit so the canonical manifest pointer is advanced at most
// once per RPC.
committed bool
// commitConflictErr is the error registered via SetCommitConflictError. Commit
// returns it instead of ErrManifestPointerConflict when CAS fails on the
// manifest pointer.
commitConflictErr error
// afterCommitMu guards afterCommit, afterCommitClosed and commitHandle. Unlike the rest
// of the cache, they are used from other goroutines than the RPC's: AfterCommit and
// CacheByCommitHandle are called by the hook requests that Git sends while the RPC waits
// for it.
afterCommitMu sync.Mutex
// afterCommit are the functions registered with AfterCommit, in registration order.
afterCommit []func()
// afterCommitClosed is set once Commit or Cleanup took the registered functions, after
// which AfterCommit refuses new ones.
afterCommitClosed bool
// commitHandle is the handle that CommitHandle returned for this cache, if any.
commitHandle string
// unlockRepository releases the repository lock. It is set for mutators only, and only
// once the cache has been prepared. Preparing a cache without cleaning it up leaves the
// repository locked for the lifetime of the process.
unlockRepository func()
// metrics records prepare latency and commit outcomes. It is nil for caches
// built outside the CacheFactory (e.g. in tests), in which case recording is
// skipped.
metrics *cacheMetrics
}
// Interface compliance check.
var _ Cache = &LocalCache{}
// NewLocalCache returns a new MVCCCache pointer.
func NewLocalCache(rs ObjectStorage, runtimeDir string, storagePath string, repositoryID string, readOnly bool) *LocalCache {
return &LocalCache{
rs: rs,
repositoryID: repositoryID,
runtimeDir: runtimeDir,
storagePath: storagePath,
environments: make(map[string]string),
readOnly: readOnly,
readyDone: make(chan struct{}),
prepareDone: make(chan struct{}),
}
}
// Prepare calls PrepareWithManifest under the hood with an empty manifest hash, which semantically
// means to warm the cache using the current manifest pointer.
func (m *LocalCache) Prepare(ctx context.Context) {
m.PrepareWithManifest(ctx, latestManifestPointer)
}
// PrepareWithManifest kicks off cache warming in the background and returns immediately. The result
// of the warm-up (nil on success, or the first error encountered) is delivered to the ready channel,
// which WaitUntilReady consumes.
// Repeated calls are no-ops: the work runs at most once per cache instance.
// The hash parameter must be the manifest hash to use for cache warming.
//
// For a mutator the warm-up first locks the repository against garbage collection, and it stays
// locked until Cleanup is called. That covers the window in which Git has written an artifact but
// Commit has not published the manifest referencing it. Callers must therefore always pair Prepare
// with Cleanup, or the repository would never be garbage collected again.
//
// Failing to lock the repository is reported like any other warm-up failure, so that this call
// stays asynchronous even when garbage collection is holding the repository.
func (m *LocalCache) PrepareWithManifest(ctx context.Context, hash string) {
m.prepareOnce.Do(func() {
go func() {
defer close(m.prepareDone)
if !m.readOnly {
unlock, err := LockRepositoryForWrite(ctx, m.repositoryPath())
if err != nil {
m.prepareErr = err
return
}
m.unlockRepository = unlock
}
m.prepareErr = m.prepare(ctx, hash)
}()
})
}
// WaitUntilReady blocks until one of those two conditions occur:
// 1. The context is canceled
// 2. The ready phase is done.
//
// If the ready phase is done, the same error returned in the prepare
// phase is returned every time. This makes it safe to call this function
// multiple time. Unless the context gets canceled, the same error as returned
// by the prepare phase will be returned every single time.
func (m *LocalCache) WaitUntilReady(ctx context.Context) error {
m.readyOnce.Do(func() {
go func() {
<-m.prepareDone
close(m.readyDone)
}()
})
select {
case <-ctx.Done():
return ctx.Err()
case <-m.readyDone:
return m.prepareErr
}
}
// Commit publishes any MVCC artifacts that have been produced by the LocalCache, and
// updates the repository to point to the latest manifest.
func (m *LocalCache) Commit(ctx context.Context) (returnedErr error) {
if m.readOnly {
return nil
}
// Count the commit outcome. Read-only RPCs return above and are not counted.
// The conflict is flagged at the compare-and-swap below rather than derived
// from returnedErr, because a caller-registered conflict error (see
// SetCommitConflictError) no longer matches ErrManifestPointerConflict.
var conflict bool
defer func() { m.recordCommit(returnedErr, conflict) }()
// The handler has returned, so nothing registers functions anymore. They run only
// once the changes are published, and are dropped on every error path below.
afterCommit := m.takeAfterCommit()
// Wait for warming to finish before reading environments, which the background
// prepare goroutine writes. A background context is used so the wait ignores request
// cancellation.
if err := m.WaitUntilReady(context.Background()); err != nil {
return err
}
if m.ManifestPath() == "" {
return fmt.Errorf("empty GIT_MVCC_MANIFEST_PATH")
}
// Assert the at-most-once property: advancing the canonical pointer more than
// once for a single RPC would result in a torn write.
if m.committed {
return fmt.Errorf("mvcc: commit called more than once")
}
m.committed = true
newHashBytes, err := os.ReadFile(m.ManifestPath())
if err != nil {
return fmt.Errorf("read from manifest path: %w", err)
}
newHash := strings.TrimSuffix(string(newHashBytes), "\n")
if newHash == m.base.Hash {
runAfterCommit(afterCommit)
return nil // no changes, nothing to publish
}
// Upload the new artifacts before advancing the pointer so the pointer never
// references an artifact that is not yet durable.
if err := m.putNewArtifacts(ctx, newHash); err != nil {
return fmt.Errorf("put new artifacts: %w", err)
}
// The functions registered with AfterCommit run in the order in which the RPCs of this
// process advance the pointer. An RPC that builds on the changes of another one can only
// commit after it, but without this lock its functions could still run first.
if len(afterCommit) > 0 {
lock, dropReference := repositoryLocks.acquire(m.repositoryPath())
lock.commitOrder.Lock()
defer func() {
lock.commitOrder.Unlock()
dropReference()
}()
}
// Atomically advance the canonical manifest pointer.
if err := m.rs.Commit(ctx, m.repositoryID, m.base, newHash); err != nil {
if errors.Is(err, ErrManifestPointerConflict) {
conflict = true
if m.commitConflictErr != nil {
// Carry through the original error message.
return structerr.New("%w: %s", m.commitConflictErr, err.Error())
}
}
return err
}
runAfterCommit(afterCommit)
return nil
}
// AfterCommit registers fn to run once Commit has published the changes of the RPC, see
// Cache.AfterCommit.
func (m *LocalCache) AfterCommit(fn func()) bool {
if m.readOnly {
return false
}
m.afterCommitMu.Lock()
defer m.afterCommitMu.Unlock()
if m.afterCommitClosed {
return false
}
m.afterCommit = append(m.afterCommit, fn)
return true
}
// takeAfterCommit returns the functions registered with AfterCommit and refuses new ones from
// now on. It also unregisters the commit handle of the cache, as nothing can be registered
// through it anymore.
func (m *LocalCache) takeAfterCommit() []func() {
m.afterCommitMu.Lock()
defer m.afterCommitMu.Unlock()
m.afterCommitClosed = true
if m.commitHandle != "" {
commitHandles.Delete(m.commitHandle)
}
afterCommit := m.afterCommit
m.afterCommit = nil
return afterCommit
}
// runAfterCommit runs the given functions in order.
func runAfterCommit(afterCommit []func()) {
for _, fn := range afterCommit {
fn()
}
}
// commitHandles maps the handles returned by CommitHandle to their caches.
var commitHandles sync.Map
// CommitHandle returns a handle under which CacheByCommitHandle finds the mutator cache c until
// it commits or is cleaned up. The RPC passes it to the processes it spawns, such as the
// reference-transaction hooks of Git, so that the requests those processes send back to the
// server can register functions with AfterCommit on the RPC's cache. It returns an empty string
// for read-only caches and for caches that have no handles.
func CommitHandle(c Cache) string {
cache, ok := c.(*LocalCache)
if !ok || cache.readOnly {
return ""
}
cache.afterCommitMu.Lock()
defer cache.afterCommitMu.Unlock()
if cache.afterCommitClosed {
return ""
}
if cache.commitHandle == "" {
cache.commitHandle = rand.Text()
commitHandles.Store(cache.commitHandle, cache)
}
return cache.commitHandle
}
// CacheByCommitHandle returns the cache that CommitHandle returned the handle for, or nil when
// that cache has committed or been cleaned up, or never existed in this process.
func CacheByCommitHandle(handle string) Cache {
if handle == "" {
return nil
}
cache, ok := commitHandles.Load(handle)
if !ok {
return nil
}
return cache.(*LocalCache)
}
// SetCommitConflictError registers the error Commit returns instead of
// ErrManifestPointerConflict when its compare-and-swap fails.
func (m *LocalCache) SetCommitConflictError(err error) {
m.commitConflictErr = err
}
// Cleanup removes temporary resources created by the LocalCache and unlocks the repository. It
// must be called once the RPC no longer touches the repository, which for a mutator means after
// Commit has published the new manifest.
func (m *LocalCache) Cleanup() {
// Claim the warm-up if it never started, so that waiting for it below cannot block on a
// channel that nothing is going to close. Preparing a cache that has been cleaned up is a
// bug in the caller, so making that a no-op costs nothing.
m.prepareOnce.Do(func() { close(m.prepareDone) })
// Functions that a Commit did not take belong to an RPC whose changes are thrown away.
_ = m.takeAfterCommit()
// Wait for warming to finish before reading environments, which the background
// prepare goroutine writes. Otherwise the deferred Cleanup in the mvcc interceptor
// races with prepare when the handler returns before warming completes. A background
// context is used so the wait ignores request cancellation; the goroutine may still
// be writing environments even once the RPC is done.
_ = m.WaitUntilReady(context.Background())
if path := m.ManifestPath(); path != "" {
_ = os.Remove(path)
}
if m.unlockRepository != nil {
m.unlockRepository()
}
}
// repositoryPath returns the absolute path of the repository this cache operates on. It is the
// identity the repository lock is taken under, so it must match what garbage collection uses.
func (m *LocalCache) repositoryPath() string {
return filepath.Join(m.storagePath, m.repositoryID)
}
// Environment returns the list of environment variables that must be passed to
// each Git commands executed on the repository when using this MVCC cache.
func (m *LocalCache) Environment() []EnvVar {
var result []EnvVar
for k, v := range m.environments {
result = append(result, EnvVar{Name: k, Value: v})
}
return result
}
// BaseManifestPointer returns the manifest pointer resolved at
// the start of the RPC.
func (m *LocalCache) BaseManifestPointer() string {
return m.base.Hash
}
// ReadOnly returns true of the RPC is an accessor RPC.
// False otherwise.
func (m *LocalCache) ReadOnly() bool {
return m.readOnly
}
// WalkManifest visits every artifact of the base manifest that m was prepared
// against. The resulting Artifact.Path values are absolute filepaths on disk.
func (m *LocalCache) WalkManifest(ctx context.Context, visitor ArtifactVisitor) error {
if err := m.WaitUntilReady(ctx); err != nil {
return fmt.Errorf("wait until ready: %w", err)
}
return m.baseManifest.walk(ctx, filepath.Join(m.storagePath, m.repositoryID, mvccCacheDir), visitor)
}
// ManifestHash is a helper function to retrieve the manifest hash
// used for a read-only RPC. It returns an empty string when not set
// or when the RPC is a mutator one. This function is not part of the
// Cache interface, and is mostly used for testing.
func (m *LocalCache) ManifestHash() string {
return m.environments["GIT_MVCC_MANIFEST"]
}
// ManifestPath is a helper function to retrieve the manifest path
// used for a mutator RPC. It returns an empty string when not set
// or when the RPC is an accessor one. This function is not part of the
// Cache interface, and is mostly used for testing.
func (m *LocalCache) ManifestPath() string {
return m.environments["GIT_MVCC_MANIFEST_PATH"]
}
// prepare resolves the manifest pointer, then downloads every reftable it
// references from durable storage into the local cache directory.
func (m *LocalCache) prepare(ctx context.Context, hash string) (err error) {
start := time.Now()
defer func() { m.recordPrepare(start, err) }()
// base carries the generation when the pointer was resolved from durable
// storage; a pin (explicit hash) leaves the generation unknown so Commit
// falls back to reading the head.
var base ManifestPointer
// An empty hash means to fetch the latest pointer.
if hash == latestManifestPointer {
// Read the manifest pointer from the RemoteStorage
base, err = m.rs.ReadManifestPointer(ctx, m.repositoryID)
if err != nil {
return fmt.Errorf("read manifest pointer: %w", err)
}
hash = base.Hash
} else {
base = ManifestPointer{Hash: hash}
}
manifest, err := m.prepareManifestFile(ctx, hash)
if err != nil {
return fmt.Errorf("prepare manifest file: %w", err)
}
m.base = base
m.baseManifest = manifest
// Warm the cache. We only care about reftables for now. Objects will
// be taken care of at a later stage.
err = m.prepareArtifacts(ctx, manifest.RefPaths())
if err != nil {
return fmt.Errorf("prepare artifacts: %w", err)
}
// Prepare environment variables
if m.readOnly {
m.environments["GIT_MVCC_MANIFEST"] = hash
} else {
// For mutator RPCs, we create a temp manifest pointer file that points to the manifest file
tmpManifestFileDir := filepath.Join(m.runtimeDir, mvccCacheDir, m.repositoryID)
err = os.MkdirAll(tmpManifestFileDir, 0o755)
if err != nil {
return fmt.Errorf("make manifest temp dir: %w", err)
}
tmpManifestFile, err := os.CreateTemp(tmpManifestFileDir, "manifest-*")
if err != nil {
return fmt.Errorf("create temp manifest file: %w", err)
}
defer func() { _ = tmpManifestFile.Close() }()
if _, err := tmpManifestFile.WriteString(hash); err != nil {
return fmt.Errorf("write temp manifest file: %w", err)
}
m.environments["GIT_MVCC_MANIFEST_PATH"] = tmpManifestFile.Name()
}
return nil
}
// prepareManifestFile will read the manifest file defined by `hash` and return its parsed form.
// If the manifest file already exist on the local filesystem, the file is read from the filesystem.
// Else, the manifest file is fetched from ObjectStorage and then created on the local filesystem for future use.
func (m *LocalCache) prepareManifestFile(ctx context.Context, hash string) (manifest *Manifest, err error) {
// Construct the manifest path
manifestFilePath := filepath.Join(m.storagePath, m.repositoryID, mvccCacheDir, "manifests", hash)
// Verify if it already exists on the local filesystem or not
existLocally, err := fileExist(manifestFilePath)
if err != nil {
return nil, fmt.Errorf("check if file exists: %w", err)
}
if !existLocally {
// If it does not exist, read content from ObjectStorage
manifestReader, err := m.rs.ReadManifest(ctx, m.repositoryID, hash)
if err != nil {
return nil, fmt.Errorf("read manifest file: %w", err)
}
defer func() {
closeErr := manifestReader.Close()
err = errors.Join(err, closeErr)
}()
// The manifest is small and is parsed in memory anyway. It is verified against its
// name before it is written to the local cache, so that a corrupt or substituted
// object in the storage is never cached under the name it was requested with.
manifestBody, err := io.ReadAll(manifestReader)
if err != nil {
return nil, fmt.Errorf("read manifest body: %w", err)
}
if err := verifyManifestName(manifestBody, hash); err != nil {
return nil, fmt.Errorf("verify manifest: %w", err)
}
// Create the manifest file on the local file system atomically. This is a protection against
// potential concurrent requests trying to create the same manifest file.
if err = createFileAtomic(ctx, manifestFilePath, bytes.NewReader(manifestBody)); err != nil {
return nil, fmt.Errorf("write manifest: %w", err)
}
}
// Now that it exist on disk, let's open the file.
manifestFile, err := os.Open(manifestFilePath)
if err != nil {
return nil, fmt.Errorf("opening manifest %s: %w", manifestFilePath, err)
}
// Read the content in memory. These files should not be that large.
// The Manifest parsing logic needs the whole file at once to parse it.
manifestContent := bytes.Buffer{}
_, err = manifestContent.ReadFrom(manifestFile)
if err != nil {
return nil, fmt.Errorf("reading manifest: %w", err)
}
// Return the parsed manifest
return ParseManifest(manifestContent.Bytes())
}
func (m *LocalCache) prepareArtifacts(ctx context.Context, artifacts []string) error {
const numWorkers = 5
// errgroup gives us a derived context that is canceled as soon as any worker
// returns an error, which unblocks the feeder below and the other workers.
g, ctx := errgroup.WithContext(ctx)
// Feed each artifact into a channel consumed by all workers below. The
// feeder stops early if the context is canceled (error or RPC cancellation).
feedCh := make(chan string)
g.Go(func() error {
defer close(feedCh)
for _, artifact := range artifacts {
select {
case <-ctx.Done():
return ctx.Err()
case feedCh <- artifact:
}
}
return nil
})
for i := 0; i < numWorkers; i++ {
g.Go(func() error {
for artifact := range feedCh {
if err := ctx.Err(); err != nil {
return err
}
artifactLocalPath := filepath.Join(m.storagePath, m.repositoryID, mvccCacheDir, artifact)
exist, err := fileExist(artifactLocalPath)
if err != nil {
return fmt.Errorf("check if file %s exist: %w", artifactLocalPath, err)
}
// If the file already exists, there is nothing else to do.
if exist {
continue
}
readArtifactOp := func(ctx context.Context, r io.ReadCloser) error {
return createFileAtomic(ctx, artifactLocalPath, r)
}
// readArtifactOp will be executed only once across concurrent requests
// for the same artifact. The request that triggers the fetch is called
// the leader. For the followers (other concurrent request) they will
// block on `ReadArtifact` until either the fetch completes or the request
// times out.
err = m.rs.ReadArtifact(ctx, m.repositoryID, artifact, readArtifactOp)
if err != nil {
return fmt.Errorf("read artifact %q: %w", artifact, err)
}
}
return nil
})
}
return g.Wait()
}
func (m *LocalCache) putNewArtifacts(ctx context.Context, newHash string) error {
manifestFile, err := os.Open(filepath.Join(m.storagePath, m.repositoryID, mvccCacheDir, manifestDirectory, newHash))
if err != nil {
return fmt.Errorf("opening manifest: %w", err)
}
defer func() {
_ = manifestFile.Close()
}()
manifestContent := bytes.Buffer{}
_, err = manifestContent.ReadFrom(manifestFile)
if err != nil {
return fmt.Errorf("reading manifest: %w", err)
}
newManifest, err := ParseManifest(manifestContent.Bytes())
if err != nil {
return fmt.Errorf("parse new manifest: %w", err)
}
baseSet := make(map[string]struct{}, len(m.baseManifest.Paths))
for _, p := range m.baseManifest.Paths {
baseSet[p] = struct{}{}
}
newArtifacts := make([]string, 0, len(newManifest.Paths))
manifestKey := filepath.Join(manifestDirectory, newHash)
newArtifacts = append(newArtifacts, manifestKey)
for _, p := range newManifest.Paths {
if _, ok := baseSet[p]; !ok {
newArtifacts = append(newArtifacts, p)
}
}
for _, p := range newArtifacts {
localPath := filepath.Join(m.storagePath, m.repositoryID, mvccCacheDir, p)
f, err := os.Open(localPath)
if err != nil {
return fmt.Errorf("read artifact before uploading: %w", err)
}
if err := m.rs.PutArtifact(ctx, m.repositoryID, p, f); err != nil {
_ = f.Close()
return fmt.Errorf("upload artifact %s: %w", p, err)
}
_ = f.Close()
}
return nil
}
// verifyManifestName checks that the manifest body hashes to the name it was requested
// with. A manifest is content-addressed: its name is the hex SHA-256 of its payload, which
// is the body without the 32-byte trailer. The trailer itself is checked by ParseManifest.
func verifyManifestName(body []byte, name string) error {
if len(body) < trailerSize {
return fmt.Errorf("manifest %s is too short (%d bytes)", name, len(body))
}
payloadEnd := len(body) - trailerSize
sum := sha256.Sum256(body[:payloadEnd])
if got := hex.EncodeToString(sum[:]); got != name {
return fmt.Errorf("manifest content hashes to %s, expected %s", got, name)
}
return nil
}
func fileExist(path string) (bool, error) {
_, err := os.Stat(path)
if err != nil {
if errors.Is(err, os.ErrNotExist) {
return false, nil
}
return false, fmt.Errorf("stat manifest: %w", err)
}
return true, nil
}
func createFileAtomic(ctx context.Context, path string, content io.Reader) error {
w, err := safe.NewFileWriter(path)
if err != nil {
return err
}
defer func() { _ = w.Close() }()
if _, err := io.Copy(w, content); err != nil { // w implements io.Writer
return err
}
// Atomically rename temp file -> absPath.
if err := w.Commit(ctx); err != nil {
return err
}
return nil
}