Skip to content

Git designed for AI agents.

corestore: owned pack keys (PLAN7 WP72a)

Federico Elia42882a298b15
diff --git a/internal/gitaly/service/storagecore/import.go b/internal/gitaly/service/storagecore/import.go
index b0cb2a8..b36e087 100644
--- a/internal/gitaly/service/storagecore/import.go
+++ b/internal/gitaly/service/storagecore/import.go
@@ -149,7 +149,7 @@ func (s *server) ImportRepository(ctx context.Context, req *gitalypb.ImportRepos
var packKey string
var packBytes int64
if !pack.Empty() {
- packKey = corestore.PackKey(reg.repo.ID, pack.SHA256)
+ packKey = corestore.NewPackKey(reg.repo.ID, pack.SHA256)
if err := uploadPack(ctx, core, packKey, pack); err != nil {
return nil, structerr.NewUnavailable("storage core: upload pack: %w; %s", err, resumeHint)
}
diff --git a/internal/gitaly/service/storagecore/storagecore_test.go b/internal/gitaly/service/storagecore/storagecore_test.go
index 7943d37..da814f6 100644
--- a/internal/gitaly/service/storagecore/storagecore_test.go
+++ b/internal/gitaly/service/storagecore/storagecore_test.go
@@ -182,7 +182,7 @@ func (h *harness) commitElsewhere(repoID int64, ref string) git.ObjectID {
pack, err := gitops.BuildPack(h.ctx, localrepo.NewTestRepo(h.t, h.cfg, otherProto), []git.ObjectID{oid}, h.t.TempDir())
require.NoError(h.t, err)
require.False(h.t, pack.Empty())
- key := corestore.PackKey(repoID, pack.SHA256)
+ key := corestore.NewPackKey(repoID, pack.SHA256)
f, err := pack.Open()
require.NoError(h.t, err)
require.NoError(h.t, h.core.Blobs.Put(h.ctx, key, f))
diff --git a/internal/gitaly/service/storagecore/verify_objects_test.go b/internal/gitaly/service/storagecore/verify_objects_test.go
index aeea44a..3e7e702 100644
--- a/internal/gitaly/service/storagecore/verify_objects_test.go
+++ b/internal/gitaly/service/storagecore/verify_objects_test.go
@@ -39,7 +39,7 @@ func TestVerifyRepositoryReportsObjectsMissingFromStorage(t *testing.T) {
pack, err := gitops.BuildPack(h.ctx, localrepo.NewTestRepo(t, h.cfg, h.repo), []git.ObjectID{c}, t.TempDir())
require.NoError(t, err)
require.False(t, pack.Empty())
- key := corestore.PackKey(repoID, pack.SHA256)
+ key := corestore.NewPackKey(repoID, pack.SHA256)
f, err := pack.Open()
require.NoError(t, err)
require.NoError(t, h.core.Blobs.Put(h.ctx, key, f))
diff --git a/internal/graft/corestore/blobstore/blobstore_test.go b/internal/graft/corestore/blobstore/blobstore_test.go
index 9865377..2436b48 100644
--- a/internal/graft/corestore/blobstore/blobstore_test.go
+++ b/internal/graft/corestore/blobstore/blobstore_test.go
@@ -45,7 +45,7 @@ func openStore(t *testing.T, bucketURL string) *blobstore.Store {
}
func packKeyOf(content []byte) string {
- return corestore.PackKey(1, sha256.Sum256(content))
+ return corestore.NewPackKey(1, sha256.Sum256(content))
}
// failingReader fails the test when it is read.
@@ -191,7 +191,7 @@ func TestGetMissingKey(t *testing.T) {
require.NoError(t, err)
require.False(t, exists)
- _, err = s.Get(ctx, corestore.PackKey(7, sha256.Sum256([]byte("absent"))))
+ _, err = s.Get(ctx, corestore.NewPackKey(7, sha256.Sum256([]byte("absent"))))
require.ErrorIs(t, err, corestore.ErrBlobNotFound)
})
}
diff --git a/internal/graft/corestore/blobstore/durable_internal_test.go b/internal/graft/corestore/blobstore/durable_internal_test.go
index 349558f..3c567db 100644
--- a/internal/graft/corestore/blobstore/durable_internal_test.go
+++ b/internal/graft/corestore/blobstore/durable_internal_test.go
@@ -30,7 +30,7 @@ func openFileStoreForTest(t *testing.T, dir string, hook func(path string, isDir
func TestPutSyncsFileBeforeCommitAndSyncsDirectories(t *testing.T) {
dir := t.TempDir()
content := []byte("pack content, synced before the commit")
- key := corestore.PackKey(1, sha256.Sum256(content))
+ key := corestore.NewPackKey(1, sha256.Sum256(content))
final := filepath.Join(dir, filepath.FromSlash(key))
var fileSyncs int
@@ -73,7 +73,7 @@ func TestPutSyncsFileBeforeCommitAndSyncsDirectories(t *testing.T) {
func TestPutFailedFileSyncLeavesNoObject(t *testing.T) {
dir := t.TempDir()
content := []byte("pack content whose sync fails")
- key := corestore.PackKey(1, sha256.Sum256(content))
+ key := corestore.NewPackKey(1, sha256.Sum256(content))
final := filepath.Join(dir, filepath.FromSlash(key))
syncErr := errors.New("disk on fire")
@@ -110,7 +110,7 @@ func TestPutWithoutFileSyncHookSucceeds(t *testing.T) {
ctx := testhelper.Context(t)
content := []byte("durable content")
- key := corestore.PackKey(2, sha256.Sum256(content))
+ key := corestore.NewPackKey(2, sha256.Sum256(content))
require.NoError(t, s.Put(ctx, key, bytes.NewReader(content)))
require.NoError(t, s.Put(ctx, key, bytes.NewReader(content)))
@@ -129,7 +129,7 @@ func TestPutMemBucketWithoutFileSync(t *testing.T) {
t.Cleanup(func() { syncHook = nil })
content := []byte("mem content")
- key := corestore.PackKey(3, sha256.Sum256(content))
+ key := corestore.NewPackKey(3, sha256.Sum256(content))
require.NoError(t, s.Put(testhelper.Context(t), key, bytes.NewReader(content)))
}
diff --git a/internal/graft/corestore/coretest/import.go b/internal/graft/corestore/coretest/import.go
index 6df4499..3939b81 100644
--- a/internal/graft/corestore/coretest/import.go
+++ b/internal/graft/corestore/coretest/import.go
@@ -50,7 +50,7 @@ func Import(tb testing.TB, core *corestore.Core, repo *localrepo.Repo, repoProto
var packKey string
if !pack.Empty() {
- packKey = corestore.PackKey(registered.ID, pack.SHA256)
+ packKey = corestore.NewPackKey(registered.ID, pack.SHA256)
f, err := pack.Open()
require.NoError(tb, err)
require.NoError(tb, core.Blobs.Put(ctx, packKey, f))
diff --git a/internal/graft/corestore/gate/gate_fix3_test.go b/internal/graft/corestore/gate/gate_fix3_test.go
index a4f855b..fca8f3a 100644
--- a/internal/graft/corestore/gate/gate_fix3_test.go
+++ b/internal/graft/corestore/gate/gate_fix3_test.go
@@ -112,7 +112,7 @@ func TestGateHistoryGuardWithRealSyncerOnFilesBackendCopy(t *testing.T) {
pack, err := gitops.BuildFullPack(ctx, local, t.TempDir())
require.NoError(t, err)
defer func() { require.NoError(t, pack.Remove()) }()
- packKey := corestore.PackKey(rec.ID, pack.SHA256)
+ packKey := corestore.NewPackKey(rec.ID, pack.SHA256)
rc, err := pack.Open()
require.NoError(t, err)
require.NoError(t, core.Blobs.Put(ctx, packKey, rc))
diff --git a/internal/graft/corestore/packkey_test.go b/internal/graft/corestore/packkey_test.go
new file mode 100644
index 0000000..56fb94e
--- /dev/null
+++ b/internal/graft/corestore/packkey_test.go
@@ -0,0 +1,83 @@
+package corestore
+
+import (
+ "crypto/sha256"
+ "fmt"
+ "regexp"
+ "strings"
+ "testing"
+
+ "github.com/stretchr/testify/require"
+)
+
+func TestNewPackKeyIsOwnedAndParses(t *testing.T) {
+ t.Parallel()
+
+ sum := sha256.Sum256([]byte("a pack"))
+ key := NewPackKey(42, sum)
+
+ require.Regexp(t, regexp.MustCompile(`^repos/42/packs/[0-9a-f]{64}-[0-9a-f]{32}\.pack$`), key)
+ require.True(t, IsOwnedPackKey(key))
+
+ id, got, ok := ParsePackKey(key)
+ require.True(t, ok)
+ require.Equal(t, int64(42), id)
+ require.Equal(t, sum, got)
+}
+
+func TestNewPackKeyIsUniquePerUpload(t *testing.T) {
+ t.Parallel()
+
+ // Byte-identical packs get distinct keys, so no pack object is shared
+ // between two rows (PLAN7, PACK OWNERSHIP RULE 1).
+ sum := sha256.Sum256([]byte("identical content"))
+ seen := map[string]struct{}{}
+ for range 1000 {
+ key := NewPackKey(1, sum)
+ _, dup := seen[key]
+ require.False(t, dup, "duplicate key %s", key)
+ seen[key] = struct{}{}
+ }
+}
+
+func TestParsePackKeyAcceptsLegacyKeys(t *testing.T) {
+ t.Parallel()
+
+ sum := sha256.Sum256([]byte("written by v0.1.0"))
+ legacy := fmt.Sprintf("repos/7/packs/%x.pack", sum)
+
+ id, got, ok := ParsePackKey(legacy)
+ require.True(t, ok)
+ require.Equal(t, int64(7), id)
+ require.Equal(t, sum, got)
+ require.False(t, IsOwnedPackKey(legacy), "a legacy key is readable but not owned")
+}
+
+func TestParsePackKeyRejectsNonCanonicalKeys(t *testing.T) {
+ t.Parallel()
+
+ hexSum := fmt.Sprintf("%x", sha256.Sum256([]byte("x")))
+ nonce := strings.Repeat("ab", 16)
+
+ for _, key := range []string{
+ "",
+ "repos/1/packs/" + hexSum, // no suffix
+ "repos/1/packs/" + hexSum + ".pack/extra", // trailing segment
+ "repo/1/packs/" + hexSum + ".pack", // wrong prefix
+ "repos/1/objects/" + hexSum + ".pack", // wrong directory
+ "repos/0/packs/" + hexSum + ".pack", // repository ID 0
+ "repos/-1/packs/" + hexSum + ".pack", // negative ID
+ "repos/01/packs/" + hexSum + ".pack", // leading zero
+ "repos/1/packs/" + strings.ToUpper(hexSum) + ".pack", // uppercase hex
+ "repos/1/packs/" + hexSum[:63] + ".pack", // short SHA-256
+ "repos/1/packs/" + hexSum + "-" + nonce[:31] + ".pack", // short nonce
+ "repos/1/packs/" + hexSum + "-" + nonce + "ab" + ".pack", // long nonce
+ "repos/1/packs/" + hexSum + "-" + strings.ToUpper(nonce) + ".pack", // uppercase nonce
+ "repos/1/packs/" + hexSum + "-" + strings.Repeat("zz", 16) + ".pack", // nonce not hex
+ "repos/1/packs/" + hexSum + "-" + nonce + "-" + nonce + ".pack", // two nonces
+ } {
+ _, _, ok := ParsePackKey(key)
+ require.False(t, ok, "accepted %q", key)
+ require.False(t, IsOwnedPackKey(key), "owned %q", key)
+ }
+}
diff --git a/internal/graft/corestore/syncer/syncer_import_test.go b/internal/graft/corestore/syncer/syncer_import_test.go
index 0be4e3a..2099ad2 100644
--- a/internal/graft/corestore/syncer/syncer_import_test.go
+++ b/internal/graft/corestore/syncer/syncer_import_test.go
@@ -103,7 +103,7 @@ func testSyncerIncompleteImport(t *testing.T, dialect sqlstore.Dialect, backend
pack, err := gitops.BuildFullPack(e.ctx, local, t.TempDir())
require.NoError(t, err)
defer func() { require.NoError(t, pack.Remove()) }()
- packKey := corestore.PackKey(registered.ID, pack.SHA256)
+ packKey := corestore.NewPackKey(registered.ID, pack.SHA256)
rc, err := pack.Open()
require.NoError(t, err)
require.NoError(t, e.core.Blobs.Put(e.ctx, packKey, rc))
diff --git a/internal/graft/corestore/syncer/syncer_race_test.go b/internal/graft/corestore/syncer/syncer_race_test.go
index 6438d64..da05b2d 100644
--- a/internal/graft/corestore/syncer/syncer_race_test.go
+++ b/internal/graft/corestore/syncer/syncer_race_test.go
@@ -36,7 +36,7 @@ func (e *env) commitFrom(t *testing.T, id int64, node string, src source, update
require.NoError(t, err)
defer func() { require.NoError(t, pack.Remove()) }()
if !pack.Empty() {
- in.PackKey = corestore.PackKey(id, pack.SHA256)
+ in.PackKey = corestore.NewPackKey(id, pack.SHA256)
in.PackBytes = pack.Size
rc, err := pack.Open()
require.NoError(t, err)
diff --git a/internal/graft/corestore/syncer/syncer_test.go b/internal/graft/corestore/syncer/syncer_test.go
index 314a02c..6535475 100644
--- a/internal/graft/corestore/syncer/syncer_test.go
+++ b/internal/graft/corestore/syncer/syncer_test.go
@@ -185,7 +185,7 @@ func (e *env) changeset(t *testing.T, id int64, src source, updates []corestore.
NodeID: "node-a",
}
if !pack.Empty() {
- key = corestore.PackKey(id, pack.SHA256)
+ key = corestore.NewPackKey(id, pack.SHA256)
rc, err := pack.Open()
require.NoError(t, err)
require.NoError(t, e.core.Blobs.Put(e.ctx, key, rc))
diff --git a/internal/graft/corestore/types.go b/internal/graft/corestore/types.go
index cd71e59..af22f51 100644
--- a/internal/graft/corestore/types.go
+++ b/internal/graft/corestore/types.go
@@ -16,6 +16,7 @@
package corestore
import (
+ "crypto/rand"
"encoding/hex"
"errors"
"fmt"
@@ -186,41 +187,85 @@ var (
ErrBlobCorrupt = errors.New("storage core: blob content does not match its key")
)
-// PackKey returns the bucket key of a pack of the repository with the given
-// SHA-256 of its bytes: "repos/<repo_id>/packs/<hex sha256>.pack".
-func PackKey(repoID int64, sha256 [32]byte) string {
- return fmt.Sprintf("repos/%d/packs/%x.pack", repoID, sha256)
+// packNonceSize is the number of random bytes that make a pack key unique.
+const packNonceSize = 16
+
+// NewPackKey returns a new owned bucket key for a pack of the repository whose
+// bytes have the given SHA-256: "repos/<repo_id>/packs/<hex sha256>-<hex
+// nonce>.pack", with a random nonce drawn for this one upload. Every upload uses a
+// new key, so a pack object is never shared between two rows and a write never
+// relies on an object it did not write (doc/graft/PLAN7.md, PACK OWNERSHIP RULE).
+func NewPackKey(repoID int64, sum [32]byte) string {
+ var nonce [packNonceSize]byte
+ // crypto/rand.Read never fails: it aborts the program instead.
+ _, _ = rand.Read(nonce[:])
+ return ownedPackKey(repoID, sum, nonce)
+}
+
+func ownedPackKey(repoID int64, sum [32]byte, nonce [packNonceSize]byte) string {
+ return fmt.Sprintf("repos/%d/packs/%x-%x.pack", repoID, sum, nonce)
}
-// ParsePackKey returns the repository ID and the SHA-256 encoded in a key
-// made by PackKey, and false when key is not exactly such a key.
+func legacyPackKey(repoID int64, sum [32]byte) string {
+ return fmt.Sprintf("repos/%d/packs/%x.pack", repoID, sum)
+}
+
+// ParsePackKey returns the repository ID and the SHA-256 encoded in an owned
+// pack key (NewPackKey) or in a legacy one, "repos/<repo_id>/packs/<hex
+// sha256>.pack", written by v0.1.0 and still referenced by its changesets. It
+// returns false for anything else, including non-canonical spellings (uppercase
+// hex, leading zeros).
func ParsePackKey(key string) (repoID int64, sum [32]byte, ok bool) {
+ repoID, sum, _, ok = parsePackKey(key)
+ return repoID, sum, ok
+}
+
+// IsOwnedPackKey reports whether key is an owned pack key, the only kind a
+// writer may produce.
+func IsOwnedPackKey(key string) bool {
+ _, _, owned, ok := parsePackKey(key)
+ return ok && owned
+}
+
+func parsePackKey(key string) (repoID int64, sum [32]byte, owned, ok bool) {
rest, found := strings.CutPrefix(key, "repos/")
if !found {
- return 0, sum, false
+ return 0, sum, false, false
}
idPart, rest, found := strings.Cut(rest, "/packs/")
if !found {
- return 0, sum, false
+ return 0, sum, false, false
+ }
+ name, found := strings.CutSuffix(rest, ".pack")
+ if !found {
+ return 0, sum, false, false
}
- hexSum, found := strings.CutSuffix(rest, ".pack")
- if !found || len(hexSum) != 64 {
- return 0, sum, false
+ hexSum, hexNonce, owned := strings.Cut(name, "-")
+ if len(hexSum) != 2*len(sum) || (owned && len(hexNonce) != 2*packNonceSize) {
+ return 0, sum, false, false
}
+
id, err := strconv.ParseInt(idPart, 10, 64)
if err != nil || id <= 0 {
- return 0, sum, false
+ return 0, sum, false, false
}
- decoded, err := hex.DecodeString(hexSum)
- if err != nil {
- return 0, sum, false
+ if _, err := hex.Decode(sum[:], []byte(hexSum)); err != nil {
+ return 0, [32]byte{}, false, false
}
- copy(sum[:], decoded)
+
// Only the canonical spelling is a pack key (lowercase hex, no leading zeros).
- if PackKey(id, sum) != key {
- return 0, [32]byte{}, false
+ canonical := legacyPackKey(id, sum)
+ if owned {
+ var nonce [packNonceSize]byte
+ if _, err := hex.Decode(nonce[:], []byte(hexNonce)); err != nil {
+ return 0, [32]byte{}, false, false
+ }
+ canonical = ownedPackKey(id, sum, nonce)
+ }
+ if canonical != key {
+ return 0, [32]byte{}, false, false
}
- return id, sum, true
+ return id, sum, owned, true
}
// IsChangelogRef reports whether updates of the reference are recorded in the
diff --git a/internal/graft/corestore/voter/server_test.go b/internal/graft/corestore/voter/server_test.go
index f985223..c0c24d6 100644
--- a/internal/graft/corestore/voter/server_test.go
+++ b/internal/graft/corestore/voter/server_test.go
@@ -148,7 +148,7 @@ func TestAuthoritativeServer(t *testing.T) {
packFile, err := pack.Open()
require.NoError(t, err)
- key := corestore.PackKey(repoID, pack.SHA256)
+ key := corestore.NewPackKey(repoID, pack.SHA256)
require.NoError(t, core.Blobs.Put(ctx, key, packFile))
require.NoError(t, packFile.Close())
diff --git a/internal/graft/corestore/voter/voter.go b/internal/graft/corestore/voter/voter.go
index 7022c8a..059e0e0 100644
--- a/internal/graft/corestore/voter/voter.go
+++ b/internal/graft/corestore/voter/voter.go
@@ -674,7 +674,7 @@ func (v *Voter) buildAndUpload(ctx context.Context, repo *localrepo.Repo, storag
return uploadedPack{}, nil
}
- key := corestore.PackKey(repoID, pack.SHA256)
+ key := corestore.NewPackKey(repoID, pack.SHA256)
if err := v.uploadPack(ctx, key, pack); err != nil {
return uploadedPack{}, err
}