Git designed for AI agents.
corestore: owned pack keys (PLAN7 WP72a)
Federico Elia42882a298b15
diff --git a/internal/gitaly/service/storagecore/import.go b/internal/gitaly/service/storagecore/import.goindex b0cb2a8..b36e087 100644--- a/internal/gitaly/service/storagecore/import.go+++ b/internal/gitaly/service/storagecore/import.go@@ -149,7 +149,7 @@ func (s *server) ImportRepository(ctx context.Context, req *gitalypb.ImportReposvar packKey stringvar packBytes int64if !pack.Empty() {- packKey = corestore.PackKey(reg.repo.ID, pack.SHA256)+ packKey = corestore.NewPackKey(reg.repo.ID, pack.SHA256)if err := uploadPack(ctx, core, packKey, pack); err != nil {return nil, structerr.NewUnavailable("storage core: upload pack: %w; %s", err, resumeHint)}diff --git a/internal/gitaly/service/storagecore/storagecore_test.go b/internal/gitaly/service/storagecore/storagecore_test.goindex 7943d37..da814f6 100644--- a/internal/gitaly/service/storagecore/storagecore_test.go+++ b/internal/gitaly/service/storagecore/storagecore_test.go@@ -182,7 +182,7 @@ func (h *harness) commitElsewhere(repoID int64, ref string) git.ObjectID {pack, err := gitops.BuildPack(h.ctx, localrepo.NewTestRepo(h.t, h.cfg, otherProto), []git.ObjectID{oid}, h.t.TempDir())require.NoError(h.t, err)require.False(h.t, pack.Empty())- key := corestore.PackKey(repoID, pack.SHA256)+ key := corestore.NewPackKey(repoID, pack.SHA256)f, err := pack.Open()require.NoError(h.t, err)require.NoError(h.t, h.core.Blobs.Put(h.ctx, key, f))diff --git a/internal/gitaly/service/storagecore/verify_objects_test.go b/internal/gitaly/service/storagecore/verify_objects_test.goindex aeea44a..3e7e702 100644--- a/internal/gitaly/service/storagecore/verify_objects_test.go+++ b/internal/gitaly/service/storagecore/verify_objects_test.go@@ -39,7 +39,7 @@ func TestVerifyRepositoryReportsObjectsMissingFromStorage(t *testing.T) {pack, err := gitops.BuildPack(h.ctx, localrepo.NewTestRepo(t, h.cfg, h.repo), []git.ObjectID{c}, t.TempDir())require.NoError(t, err)require.False(t, pack.Empty())- key := corestore.PackKey(repoID, pack.SHA256)+ key := corestore.NewPackKey(repoID, pack.SHA256)f, err := pack.Open()require.NoError(t, err)require.NoError(t, h.core.Blobs.Put(h.ctx, key, f))diff --git a/internal/graft/corestore/blobstore/blobstore_test.go b/internal/graft/corestore/blobstore/blobstore_test.goindex 9865377..2436b48 100644--- a/internal/graft/corestore/blobstore/blobstore_test.go+++ b/internal/graft/corestore/blobstore/blobstore_test.go@@ -45,7 +45,7 @@ func openStore(t *testing.T, bucketURL string) *blobstore.Store {}func packKeyOf(content []byte) string {- return corestore.PackKey(1, sha256.Sum256(content))+ return corestore.NewPackKey(1, sha256.Sum256(content))}// failingReader fails the test when it is read.@@ -191,7 +191,7 @@ func TestGetMissingKey(t *testing.T) {require.NoError(t, err)require.False(t, exists)- _, err = s.Get(ctx, corestore.PackKey(7, sha256.Sum256([]byte("absent"))))+ _, err = s.Get(ctx, corestore.NewPackKey(7, sha256.Sum256([]byte("absent"))))require.ErrorIs(t, err, corestore.ErrBlobNotFound)})}diff --git a/internal/graft/corestore/blobstore/durable_internal_test.go b/internal/graft/corestore/blobstore/durable_internal_test.goindex 349558f..3c567db 100644--- a/internal/graft/corestore/blobstore/durable_internal_test.go+++ b/internal/graft/corestore/blobstore/durable_internal_test.go@@ -30,7 +30,7 @@ func openFileStoreForTest(t *testing.T, dir string, hook func(path string, isDirfunc TestPutSyncsFileBeforeCommitAndSyncsDirectories(t *testing.T) {dir := t.TempDir()content := []byte("pack content, synced before the commit")- key := corestore.PackKey(1, sha256.Sum256(content))+ key := corestore.NewPackKey(1, sha256.Sum256(content))final := filepath.Join(dir, filepath.FromSlash(key))var fileSyncs int@@ -73,7 +73,7 @@ func TestPutSyncsFileBeforeCommitAndSyncsDirectories(t *testing.T) {func TestPutFailedFileSyncLeavesNoObject(t *testing.T) {dir := t.TempDir()content := []byte("pack content whose sync fails")- key := corestore.PackKey(1, sha256.Sum256(content))+ key := corestore.NewPackKey(1, sha256.Sum256(content))final := filepath.Join(dir, filepath.FromSlash(key))syncErr := errors.New("disk on fire")@@ -110,7 +110,7 @@ func TestPutWithoutFileSyncHookSucceeds(t *testing.T) {ctx := testhelper.Context(t)content := []byte("durable content")- key := corestore.PackKey(2, sha256.Sum256(content))+ key := corestore.NewPackKey(2, sha256.Sum256(content))require.NoError(t, s.Put(ctx, key, bytes.NewReader(content)))require.NoError(t, s.Put(ctx, key, bytes.NewReader(content)))@@ -129,7 +129,7 @@ func TestPutMemBucketWithoutFileSync(t *testing.T) {t.Cleanup(func() { syncHook = nil })content := []byte("mem content")- key := corestore.PackKey(3, sha256.Sum256(content))+ key := corestore.NewPackKey(3, sha256.Sum256(content))require.NoError(t, s.Put(testhelper.Context(t), key, bytes.NewReader(content)))}diff --git a/internal/graft/corestore/coretest/import.go b/internal/graft/corestore/coretest/import.goindex 6df4499..3939b81 100644--- a/internal/graft/corestore/coretest/import.go+++ b/internal/graft/corestore/coretest/import.go@@ -50,7 +50,7 @@ func Import(tb testing.TB, core *corestore.Core, repo *localrepo.Repo, repoProtovar packKey stringif !pack.Empty() {- packKey = corestore.PackKey(registered.ID, pack.SHA256)+ packKey = corestore.NewPackKey(registered.ID, pack.SHA256)f, err := pack.Open()require.NoError(tb, err)require.NoError(tb, core.Blobs.Put(ctx, packKey, f))diff --git a/internal/graft/corestore/gate/gate_fix3_test.go b/internal/graft/corestore/gate/gate_fix3_test.goindex a4f855b..fca8f3a 100644--- a/internal/graft/corestore/gate/gate_fix3_test.go+++ b/internal/graft/corestore/gate/gate_fix3_test.go@@ -112,7 +112,7 @@ func TestGateHistoryGuardWithRealSyncerOnFilesBackendCopy(t *testing.T) {pack, err := gitops.BuildFullPack(ctx, local, t.TempDir())require.NoError(t, err)defer func() { require.NoError(t, pack.Remove()) }()- packKey := corestore.PackKey(rec.ID, pack.SHA256)+ packKey := corestore.NewPackKey(rec.ID, pack.SHA256)rc, err := pack.Open()require.NoError(t, err)require.NoError(t, core.Blobs.Put(ctx, packKey, rc))diff --git a/internal/graft/corestore/packkey_test.go b/internal/graft/corestore/packkey_test.gonew file mode 100644index 0000000..56fb94e--- /dev/null+++ b/internal/graft/corestore/packkey_test.go@@ -0,0 +1,83 @@+package corestore++import (+ "crypto/sha256"+ "fmt"+ "regexp"+ "strings"+ "testing"++ "github.com/stretchr/testify/require"+)++func TestNewPackKeyIsOwnedAndParses(t *testing.T) {+ t.Parallel()++ sum := sha256.Sum256([]byte("a pack"))+ key := NewPackKey(42, sum)++ require.Regexp(t, regexp.MustCompile(`^repos/42/packs/[0-9a-f]{64}-[0-9a-f]{32}\.pack$`), key)+ require.True(t, IsOwnedPackKey(key))++ id, got, ok := ParsePackKey(key)+ require.True(t, ok)+ require.Equal(t, int64(42), id)+ require.Equal(t, sum, got)+}++func TestNewPackKeyIsUniquePerUpload(t *testing.T) {+ t.Parallel()++ // Byte-identical packs get distinct keys, so no pack object is shared+ // between two rows (PLAN7, PACK OWNERSHIP RULE 1).+ sum := sha256.Sum256([]byte("identical content"))+ seen := map[string]struct{}{}+ for range 1000 {+ key := NewPackKey(1, sum)+ _, dup := seen[key]+ require.False(t, dup, "duplicate key %s", key)+ seen[key] = struct{}{}+ }+}++func TestParsePackKeyAcceptsLegacyKeys(t *testing.T) {+ t.Parallel()++ sum := sha256.Sum256([]byte("written by v0.1.0"))+ legacy := fmt.Sprintf("repos/7/packs/%x.pack", sum)++ id, got, ok := ParsePackKey(legacy)+ require.True(t, ok)+ require.Equal(t, int64(7), id)+ require.Equal(t, sum, got)+ require.False(t, IsOwnedPackKey(legacy), "a legacy key is readable but not owned")+}++func TestParsePackKeyRejectsNonCanonicalKeys(t *testing.T) {+ t.Parallel()++ hexSum := fmt.Sprintf("%x", sha256.Sum256([]byte("x")))+ nonce := strings.Repeat("ab", 16)++ for _, key := range []string{+ "",+ "repos/1/packs/" + hexSum, // no suffix+ "repos/1/packs/" + hexSum + ".pack/extra", // trailing segment+ "repo/1/packs/" + hexSum + ".pack", // wrong prefix+ "repos/1/objects/" + hexSum + ".pack", // wrong directory+ "repos/0/packs/" + hexSum + ".pack", // repository ID 0+ "repos/-1/packs/" + hexSum + ".pack", // negative ID+ "repos/01/packs/" + hexSum + ".pack", // leading zero+ "repos/1/packs/" + strings.ToUpper(hexSum) + ".pack", // uppercase hex+ "repos/1/packs/" + hexSum[:63] + ".pack", // short SHA-256+ "repos/1/packs/" + hexSum + "-" + nonce[:31] + ".pack", // short nonce+ "repos/1/packs/" + hexSum + "-" + nonce + "ab" + ".pack", // long nonce+ "repos/1/packs/" + hexSum + "-" + strings.ToUpper(nonce) + ".pack", // uppercase nonce+ "repos/1/packs/" + hexSum + "-" + strings.Repeat("zz", 16) + ".pack", // nonce not hex+ "repos/1/packs/" + hexSum + "-" + nonce + "-" + nonce + ".pack", // two nonces+ } {+ _, _, ok := ParsePackKey(key)+ require.False(t, ok, "accepted %q", key)+ require.False(t, IsOwnedPackKey(key), "owned %q", key)+ }+}diff --git a/internal/graft/corestore/syncer/syncer_import_test.go b/internal/graft/corestore/syncer/syncer_import_test.goindex 0be4e3a..2099ad2 100644--- a/internal/graft/corestore/syncer/syncer_import_test.go+++ b/internal/graft/corestore/syncer/syncer_import_test.go@@ -103,7 +103,7 @@ func testSyncerIncompleteImport(t *testing.T, dialect sqlstore.Dialect, backendpack, err := gitops.BuildFullPack(e.ctx, local, t.TempDir())require.NoError(t, err)defer func() { require.NoError(t, pack.Remove()) }()- packKey := corestore.PackKey(registered.ID, pack.SHA256)+ packKey := corestore.NewPackKey(registered.ID, pack.SHA256)rc, err := pack.Open()require.NoError(t, err)require.NoError(t, e.core.Blobs.Put(e.ctx, packKey, rc))diff --git a/internal/graft/corestore/syncer/syncer_race_test.go b/internal/graft/corestore/syncer/syncer_race_test.goindex 6438d64..da05b2d 100644--- a/internal/graft/corestore/syncer/syncer_race_test.go+++ b/internal/graft/corestore/syncer/syncer_race_test.go@@ -36,7 +36,7 @@ func (e *env) commitFrom(t *testing.T, id int64, node string, src source, updaterequire.NoError(t, err)defer func() { require.NoError(t, pack.Remove()) }()if !pack.Empty() {- in.PackKey = corestore.PackKey(id, pack.SHA256)+ in.PackKey = corestore.NewPackKey(id, pack.SHA256)in.PackBytes = pack.Sizerc, err := pack.Open()require.NoError(t, err)diff --git a/internal/graft/corestore/syncer/syncer_test.go b/internal/graft/corestore/syncer/syncer_test.goindex 314a02c..6535475 100644--- a/internal/graft/corestore/syncer/syncer_test.go+++ b/internal/graft/corestore/syncer/syncer_test.go@@ -185,7 +185,7 @@ func (e *env) changeset(t *testing.T, id int64, src source, updates []corestore.NodeID: "node-a",}if !pack.Empty() {- key = corestore.PackKey(id, pack.SHA256)+ key = corestore.NewPackKey(id, pack.SHA256)rc, err := pack.Open()require.NoError(t, err)require.NoError(t, e.core.Blobs.Put(e.ctx, key, rc))diff --git a/internal/graft/corestore/types.go b/internal/graft/corestore/types.goindex cd71e59..af22f51 100644--- a/internal/graft/corestore/types.go+++ b/internal/graft/corestore/types.go@@ -16,6 +16,7 @@package corestoreimport (+ "crypto/rand""encoding/hex""errors""fmt"@@ -186,41 +187,85 @@ var (ErrBlobCorrupt = errors.New("storage core: blob content does not match its key"))-// PackKey returns the bucket key of a pack of the repository with the given-// SHA-256 of its bytes: "repos/<repo_id>/packs/<hex sha256>.pack".-func PackKey(repoID int64, sha256 [32]byte) string {- return fmt.Sprintf("repos/%d/packs/%x.pack", repoID, sha256)+// packNonceSize is the number of random bytes that make a pack key unique.+const packNonceSize = 16++// NewPackKey returns a new owned bucket key for a pack of the repository whose+// bytes have the given SHA-256: "repos/<repo_id>/packs/<hex sha256>-<hex+// nonce>.pack", with a random nonce drawn for this one upload. Every upload uses a+// new key, so a pack object is never shared between two rows and a write never+// relies on an object it did not write (doc/graft/PLAN7.md, PACK OWNERSHIP RULE).+func NewPackKey(repoID int64, sum [32]byte) string {+ var nonce [packNonceSize]byte+ // crypto/rand.Read never fails: it aborts the program instead.+ _, _ = rand.Read(nonce[:])+ return ownedPackKey(repoID, sum, nonce)+}++func ownedPackKey(repoID int64, sum [32]byte, nonce [packNonceSize]byte) string {+ return fmt.Sprintf("repos/%d/packs/%x-%x.pack", repoID, sum, nonce)}-// ParsePackKey returns the repository ID and the SHA-256 encoded in a key-// made by PackKey, and false when key is not exactly such a key.+func legacyPackKey(repoID int64, sum [32]byte) string {+ return fmt.Sprintf("repos/%d/packs/%x.pack", repoID, sum)+}++// ParsePackKey returns the repository ID and the SHA-256 encoded in an owned+// pack key (NewPackKey) or in a legacy one, "repos/<repo_id>/packs/<hex+// sha256>.pack", written by v0.1.0 and still referenced by its changesets. It+// returns false for anything else, including non-canonical spellings (uppercase+// hex, leading zeros).func ParsePackKey(key string) (repoID int64, sum [32]byte, ok bool) {+ repoID, sum, _, ok = parsePackKey(key)+ return repoID, sum, ok+}++// IsOwnedPackKey reports whether key is an owned pack key, the only kind a+// writer may produce.+func IsOwnedPackKey(key string) bool {+ _, _, owned, ok := parsePackKey(key)+ return ok && owned+}++func parsePackKey(key string) (repoID int64, sum [32]byte, owned, ok bool) {rest, found := strings.CutPrefix(key, "repos/")if !found {- return 0, sum, false+ return 0, sum, false, false}idPart, rest, found := strings.Cut(rest, "/packs/")if !found {- return 0, sum, false+ return 0, sum, false, false+ }+ name, found := strings.CutSuffix(rest, ".pack")+ if !found {+ return 0, sum, false, false}- hexSum, found := strings.CutSuffix(rest, ".pack")- if !found || len(hexSum) != 64 {- return 0, sum, false+ hexSum, hexNonce, owned := strings.Cut(name, "-")+ if len(hexSum) != 2*len(sum) || (owned && len(hexNonce) != 2*packNonceSize) {+ return 0, sum, false, false}+id, err := strconv.ParseInt(idPart, 10, 64)if err != nil || id <= 0 {- return 0, sum, false+ return 0, sum, false, false}- decoded, err := hex.DecodeString(hexSum)- if err != nil {- return 0, sum, false+ if _, err := hex.Decode(sum[:], []byte(hexSum)); err != nil {+ return 0, [32]byte{}, false, false}- copy(sum[:], decoded)+// Only the canonical spelling is a pack key (lowercase hex, no leading zeros).- if PackKey(id, sum) != key {- return 0, [32]byte{}, false+ canonical := legacyPackKey(id, sum)+ if owned {+ var nonce [packNonceSize]byte+ if _, err := hex.Decode(nonce[:], []byte(hexNonce)); err != nil {+ return 0, [32]byte{}, false, false+ }+ canonical = ownedPackKey(id, sum, nonce)+ }+ if canonical != key {+ return 0, [32]byte{}, false, false}- return id, sum, true+ return id, sum, owned, true}// IsChangelogRef reports whether updates of the reference are recorded in thediff --git a/internal/graft/corestore/voter/server_test.go b/internal/graft/corestore/voter/server_test.goindex f985223..c0c24d6 100644--- a/internal/graft/corestore/voter/server_test.go+++ b/internal/graft/corestore/voter/server_test.go@@ -148,7 +148,7 @@ func TestAuthoritativeServer(t *testing.T) {packFile, err := pack.Open()require.NoError(t, err)- key := corestore.PackKey(repoID, pack.SHA256)+ key := corestore.NewPackKey(repoID, pack.SHA256)require.NoError(t, core.Blobs.Put(ctx, key, packFile))require.NoError(t, packFile.Close())diff --git a/internal/graft/corestore/voter/voter.go b/internal/graft/corestore/voter/voter.goindex 7022c8a..059e0e0 100644--- a/internal/graft/corestore/voter/voter.go+++ b/internal/graft/corestore/voter/voter.go@@ -674,7 +674,7 @@ func (v *Voter) buildAndUpload(ctx context.Context, repo *localrepo.Repo, storagreturn uploadedPack{}, nil}- key := corestore.PackKey(repoID, pack.SHA256)+ key := corestore.NewPackKey(repoID, pack.SHA256)if err := v.uploadPack(ctx, key, pack); err != nil {return uploadedPack{}, err}