From c0d02c4c23029b8a70026f5302ccb0855059c262 Mon Sep 17 00:00:00 2001 From: Jasperhino Date: Fri, 24 Jul 2026 14:53:05 +0200 Subject: [PATCH] Fix restore-keys to prefix-match on GCS like actions/cache MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit restore-keys were checked with exact object existence, so prefix fallbacks (e.g. `nx-` matching `nx-`) never hit on GCS — they only worked via the GitHub cache fallback, which is empty when saves go to GCS. Rolling caches keyed on unique values (sha/run) were therefore never restored. Primary key stays exact-match to keep `cache-hit` semantics. Restore keys now list objects by prefix and pick the newest, mirroring the upstream actions/cache contract. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01PkmyS2WfCHHdaQQRpfqvNi --- __tests__/gcsCache.test.ts | 166 +++++++++++++++++++++++++++++++++++++ dist/restore-only/index.js | 33 ++++++-- dist/restore/index.js | 33 ++++++-- dist/save-only/index.js | 33 ++++++-- dist/save/index.js | 33 ++++++-- src/utils/gcsCache.ts | 41 +++++++-- 6 files changed, 314 insertions(+), 25 deletions(-) create mode 100644 __tests__/gcsCache.test.ts diff --git a/__tests__/gcsCache.test.ts b/__tests__/gcsCache.test.ts new file mode 100644 index 0000000..2755e0d --- /dev/null +++ b/__tests__/gcsCache.test.ts @@ -0,0 +1,166 @@ +import * as cache from "@actions/cache"; +import { Storage } from "@google-cloud/storage"; + +import * as actionUtils from "../src/utils/actionUtils"; +import { restoreCache } from "../src/utils/gcsCache"; + +jest.mock("@actions/cache"); +jest.mock("@google-cloud/storage"); +jest.mock("../src/utils/actionUtils"); +jest.mock("@actions/cache/lib/internal/cacheUtils", () => ({ + getCompressionMethod: jest.fn().mockResolvedValue("zstd"), + getCacheFileName: jest.fn().mockReturnValue("cache.tzst"), + createTempDirectory: jest.fn().mockResolvedValue("/tmp/gcs-cache-test"), + getArchiveFileSizeInBytes: jest.fn().mockReturnValue(1024), + unlinkFile: jest.fn().mockResolvedValue(undefined), + resolvePaths: jest.fn().mockResolvedValue(["some/path"]) +})); +jest.mock("@actions/cache/lib/internal/tar", () => ({ + createTar: jest.fn(), + extractTar: jest.fn(), + listTar: jest.fn() +})); + +const BUCKET = "test-bucket"; +const PREFIX = "github-cache"; + +interface FakeObject { + name: string; + updated: string; +} + +function mockStorage(objects: FakeObject[]): void { + const bucketApi = { + file: (path: string) => ({ + exists: jest + .fn() + .mockResolvedValue([objects.some(o => o.name === path)]), + download: jest.fn().mockResolvedValue(undefined) + }), + getFiles: jest.fn(({ prefix }: { prefix: string }) => [ + objects + .filter(o => o.name.startsWith(prefix)) + .map(o => ({ + name: o.name, + metadata: { updated: o.updated } + })) + ]) + }; + (Storage as unknown as jest.Mock).mockImplementation(() => ({ + bucket: () => bucketApi + })); +} + +beforeEach(() => { + jest.mocked(actionUtils.isGCSAvailable).mockReturnValue(true); + jest.mocked(actionUtils.getGCSBucket).mockReturnValue(BUCKET); + jest.mocked(cache.restoreCache).mockResolvedValue(undefined); +}); + +afterEach(() => { + jest.clearAllMocks(); +}); + +test("primary key exact match returns primary key", async () => { + mockStorage([ + { + name: `${PREFIX}/nx-abc123.cache.tzst`, + updated: "2026-07-01T00:00:00Z" + } + ]); + + const result = await restoreCache(["some/path"], "nx-abc123", ["nx-"], { + lookupOnly: true + }); + + expect(result).toBe("nx-abc123"); +}); + +test("restore key prefix-matches and returns newest entry's key", async () => { + mockStorage([ + { + name: `${PREFIX}/nx-older00.cache.tzst`, + updated: "2026-07-01T00:00:00Z" + }, + { + name: `${PREFIX}/nx-newer00.cache.tzst`, + updated: "2026-07-20T00:00:00Z" + } + ]); + + const result = await restoreCache(["some/path"], "nx-abc123", ["nx-"], { + lookupOnly: true + }); + + expect(result).toBe("nx-newer00"); +}); + +test("restore key ignores objects with other compression suffix", async () => { + mockStorage([ + { + name: `${PREFIX}/nx-gzipped0.cache.tgz`, + updated: "2026-07-20T00:00:00Z" + } + ]); + + const result = await restoreCache(["some/path"], "nx-abc123", ["nx-"], { + lookupOnly: true + }); + + expect(result).toBeUndefined(); + expect(cache.restoreCache).toHaveBeenCalled(); +}); + +test("restore keys are tried in order", async () => { + mockStorage([ + { + name: `${PREFIX}/fallback-key.cache.tzst`, + updated: "2026-07-01T00:00:00Z" + }, + { + name: `${PREFIX}/preferred-key.cache.tzst`, + updated: "2026-06-01T00:00:00Z" + } + ]); + + const result = await restoreCache( + ["some/path"], + "nx-abc123", + ["preferred-", "fallback-"], + { lookupOnly: true } + ); + + expect(result).toBe("preferred-key"); +}); + +test("no match falls back to GitHub cache", async () => { + mockStorage([]); + + const result = await restoreCache(["some/path"], "nx-abc123", ["nx-"], { + lookupOnly: true + }); + + expect(result).toBeUndefined(); + expect(cache.restoreCache).toHaveBeenCalledWith( + ["some/path"], + "nx-abc123", + ["nx-"], + { lookupOnly: true }, + undefined + ); +}); + +test("keys containing slashes resolve to the full matched key", async () => { + mockStorage([ + { + name: `${PREFIX}/develop/nx-abc.cache.tzst`, + updated: "2026-07-01T00:00:00Z" + } + ]); + + const result = await restoreCache(["some/path"], "develop/nx-xyz", [ + "develop/" + ]); + + expect(result).toBe("develop/nx-abc"); +}); diff --git a/dist/restore-only/index.js b/dist/restore-only/index.js index 3fb4e9b..3ecc699 100644 --- a/dist/restore-only/index.js +++ b/dist/restore-only/index.js @@ -78653,11 +78653,34 @@ function saveToGCS(paths, key) { } function findFileOnGCS(storage, bucket, pathPrefix, keys, compressionMethod) { return __awaiter(this, void 0, void 0, function* () { - for (const key of keys) { - const gcsPath = getGCSPath(pathPrefix, key, compressionMethod); - if (yield checkFileExists(storage, bucket, gcsPath)) { - core.info(`Found file on bucket: ${bucket} with key: ${gcsPath}`); - return { key, path: gcsPath }; + const [primaryKey, ...restoreKeys] = keys; + const fileName = utils.getCacheFileName(compressionMethod); + // Primary key: exact match only. The `cache-hit` output compares the + // returned key against the primary key, so a prefix match here would + // report false hits. + const primaryPath = getGCSPath(pathPrefix, primaryKey, compressionMethod); + if (yield checkFileExists(storage, bucket, primaryPath)) { + core.info(`Found file on bucket: ${bucket} with key: ${primaryPath}`); + return { key: primaryKey, path: primaryPath }; + } + // Restore keys: prefix match, newest entry wins — mirrors the + // actions/cache restore-keys contract that callers rely on for rolling + // caches (e.g. `nx-` matching `nx-` saved by an earlier run). + for (const key of restoreKeys) { + const [files] = yield storage + .bucket(bucket) + .getFiles({ prefix: `${pathPrefix}/${key}` }); + const newest = files + .filter(file => file.name.endsWith(`.${fileName}`)) + .sort((a, b) => { + var _a, _b; + return new Date((_a = b.metadata.updated) !== null && _a !== void 0 ? _a : 0).getTime() - + new Date((_b = a.metadata.updated) !== null && _b !== void 0 ? _b : 0).getTime(); + })[0]; + if (newest) { + const matchedKey = newest.name.slice(pathPrefix.length + 1, -(fileName.length + 1)); + core.info(`Found file on bucket: ${bucket} with key: ${newest.name}`); + return { key: matchedKey, path: newest.name }; } } return undefined; diff --git a/dist/restore/index.js b/dist/restore/index.js index d436f8d..987f095 100644 --- a/dist/restore/index.js +++ b/dist/restore/index.js @@ -78653,11 +78653,34 @@ function saveToGCS(paths, key) { } function findFileOnGCS(storage, bucket, pathPrefix, keys, compressionMethod) { return __awaiter(this, void 0, void 0, function* () { - for (const key of keys) { - const gcsPath = getGCSPath(pathPrefix, key, compressionMethod); - if (yield checkFileExists(storage, bucket, gcsPath)) { - core.info(`Found file on bucket: ${bucket} with key: ${gcsPath}`); - return { key, path: gcsPath }; + const [primaryKey, ...restoreKeys] = keys; + const fileName = utils.getCacheFileName(compressionMethod); + // Primary key: exact match only. The `cache-hit` output compares the + // returned key against the primary key, so a prefix match here would + // report false hits. + const primaryPath = getGCSPath(pathPrefix, primaryKey, compressionMethod); + if (yield checkFileExists(storage, bucket, primaryPath)) { + core.info(`Found file on bucket: ${bucket} with key: ${primaryPath}`); + return { key: primaryKey, path: primaryPath }; + } + // Restore keys: prefix match, newest entry wins — mirrors the + // actions/cache restore-keys contract that callers rely on for rolling + // caches (e.g. `nx-` matching `nx-` saved by an earlier run). + for (const key of restoreKeys) { + const [files] = yield storage + .bucket(bucket) + .getFiles({ prefix: `${pathPrefix}/${key}` }); + const newest = files + .filter(file => file.name.endsWith(`.${fileName}`)) + .sort((a, b) => { + var _a, _b; + return new Date((_a = b.metadata.updated) !== null && _a !== void 0 ? _a : 0).getTime() - + new Date((_b = a.metadata.updated) !== null && _b !== void 0 ? _b : 0).getTime(); + })[0]; + if (newest) { + const matchedKey = newest.name.slice(pathPrefix.length + 1, -(fileName.length + 1)); + core.info(`Found file on bucket: ${bucket} with key: ${newest.name}`); + return { key: matchedKey, path: newest.name }; } } return undefined; diff --git a/dist/save-only/index.js b/dist/save-only/index.js index e64bfb9..38694f8 100644 --- a/dist/save-only/index.js +++ b/dist/save-only/index.js @@ -78666,11 +78666,34 @@ function saveToGCS(paths, key) { } function findFileOnGCS(storage, bucket, pathPrefix, keys, compressionMethod) { return __awaiter(this, void 0, void 0, function* () { - for (const key of keys) { - const gcsPath = getGCSPath(pathPrefix, key, compressionMethod); - if (yield checkFileExists(storage, bucket, gcsPath)) { - core.info(`Found file on bucket: ${bucket} with key: ${gcsPath}`); - return { key, path: gcsPath }; + const [primaryKey, ...restoreKeys] = keys; + const fileName = utils.getCacheFileName(compressionMethod); + // Primary key: exact match only. The `cache-hit` output compares the + // returned key against the primary key, so a prefix match here would + // report false hits. + const primaryPath = getGCSPath(pathPrefix, primaryKey, compressionMethod); + if (yield checkFileExists(storage, bucket, primaryPath)) { + core.info(`Found file on bucket: ${bucket} with key: ${primaryPath}`); + return { key: primaryKey, path: primaryPath }; + } + // Restore keys: prefix match, newest entry wins — mirrors the + // actions/cache restore-keys contract that callers rely on for rolling + // caches (e.g. `nx-` matching `nx-` saved by an earlier run). + for (const key of restoreKeys) { + const [files] = yield storage + .bucket(bucket) + .getFiles({ prefix: `${pathPrefix}/${key}` }); + const newest = files + .filter(file => file.name.endsWith(`.${fileName}`)) + .sort((a, b) => { + var _a, _b; + return new Date((_a = b.metadata.updated) !== null && _a !== void 0 ? _a : 0).getTime() - + new Date((_b = a.metadata.updated) !== null && _b !== void 0 ? _b : 0).getTime(); + })[0]; + if (newest) { + const matchedKey = newest.name.slice(pathPrefix.length + 1, -(fileName.length + 1)); + core.info(`Found file on bucket: ${bucket} with key: ${newest.name}`); + return { key: matchedKey, path: newest.name }; } } return undefined; diff --git a/dist/save/index.js b/dist/save/index.js index 6237269..eb54b54 100644 --- a/dist/save/index.js +++ b/dist/save/index.js @@ -78666,11 +78666,34 @@ function saveToGCS(paths, key) { } function findFileOnGCS(storage, bucket, pathPrefix, keys, compressionMethod) { return __awaiter(this, void 0, void 0, function* () { - for (const key of keys) { - const gcsPath = getGCSPath(pathPrefix, key, compressionMethod); - if (yield checkFileExists(storage, bucket, gcsPath)) { - core.info(`Found file on bucket: ${bucket} with key: ${gcsPath}`); - return { key, path: gcsPath }; + const [primaryKey, ...restoreKeys] = keys; + const fileName = utils.getCacheFileName(compressionMethod); + // Primary key: exact match only. The `cache-hit` output compares the + // returned key against the primary key, so a prefix match here would + // report false hits. + const primaryPath = getGCSPath(pathPrefix, primaryKey, compressionMethod); + if (yield checkFileExists(storage, bucket, primaryPath)) { + core.info(`Found file on bucket: ${bucket} with key: ${primaryPath}`); + return { key: primaryKey, path: primaryPath }; + } + // Restore keys: prefix match, newest entry wins — mirrors the + // actions/cache restore-keys contract that callers rely on for rolling + // caches (e.g. `nx-` matching `nx-` saved by an earlier run). + for (const key of restoreKeys) { + const [files] = yield storage + .bucket(bucket) + .getFiles({ prefix: `${pathPrefix}/${key}` }); + const newest = files + .filter(file => file.name.endsWith(`.${fileName}`)) + .sort((a, b) => { + var _a, _b; + return new Date((_a = b.metadata.updated) !== null && _a !== void 0 ? _a : 0).getTime() - + new Date((_b = a.metadata.updated) !== null && _b !== void 0 ? _b : 0).getTime(); + })[0]; + if (newest) { + const matchedKey = newest.name.slice(pathPrefix.length + 1, -(fileName.length + 1)); + core.info(`Found file on bucket: ${bucket} with key: ${newest.name}`); + return { key: matchedKey, path: newest.name }; } } return undefined; diff --git a/src/utils/gcsCache.ts b/src/utils/gcsCache.ts index c1ffd3e..f9b6222 100644 --- a/src/utils/gcsCache.ts +++ b/src/utils/gcsCache.ts @@ -262,11 +262,42 @@ async function findFileOnGCS( keys: string[], compressionMethod: CompressionMethod ): Promise<{ key: string; path: string } | undefined> { - for (const key of keys) { - const gcsPath = getGCSPath(pathPrefix, key, compressionMethod); - if (await checkFileExists(storage, bucket, gcsPath)) { - core.info(`Found file on bucket: ${bucket} with key: ${gcsPath}`); - return { key, path: gcsPath }; + const [primaryKey, ...restoreKeys] = keys; + const fileName = utils.getCacheFileName(compressionMethod); + + // Primary key: exact match only. The `cache-hit` output compares the + // returned key against the primary key, so a prefix match here would + // report false hits. + const primaryPath = getGCSPath(pathPrefix, primaryKey, compressionMethod); + if (await checkFileExists(storage, bucket, primaryPath)) { + core.info(`Found file on bucket: ${bucket} with key: ${primaryPath}`); + return { key: primaryKey, path: primaryPath }; + } + + // Restore keys: prefix match, newest entry wins — mirrors the + // actions/cache restore-keys contract that callers rely on for rolling + // caches (e.g. `nx-` matching `nx-` saved by an earlier run). + for (const key of restoreKeys) { + const [files] = await storage + .bucket(bucket) + .getFiles({ prefix: `${pathPrefix}/${key}` }); + const newest = files + .filter(file => file.name.endsWith(`.${fileName}`)) + .sort( + (a, b) => + new Date(b.metadata.updated ?? 0).getTime() - + new Date(a.metadata.updated ?? 0).getTime() + )[0]; + + if (newest) { + const matchedKey = newest.name.slice( + pathPrefix.length + 1, + -(fileName.length + 1) + ); + core.info( + `Found file on bucket: ${bucket} with key: ${newest.name}` + ); + return { key: matchedKey, path: newest.name }; } } return undefined;