diff --git a/README.md b/README.md index 6cb71e7..bc3dcb7 100644 --- a/README.md +++ b/README.md @@ -103,6 +103,17 @@ If you are using a `self-hosted` Windows runner, `GNU tar` and `zstd` are requir See [Skipping steps based on cache-hit](#skipping-steps-based-on-cache-hit) for info on using this output +### Google Cloud Storage backend + +When `gcs-bucket` (or the `CULA_CACHE_GCS_BUCKET` / `CONFIGURED_GCS_BUCKET` environment variable) is set, GCS is the primary backend and the GitHub cache is the fallback: + +* **Restore** looks in GCS first (primary key, then `restore-keys`) and only then in the GitHub cache. The matched key is reported the same way for both, so `cache-hit` keeps its meaning. +* **Save** always goes to GCS. If the GCS upload fails, the entry is saved to the GitHub cache instead. +* **A hit served by the GitHub cache is written to GCS anyway.** Without this, a GitHub-only entry (from an earlier GCS failure) would satisfy every later job and GCS would never receive the key. +* **The `path` input is remembered from the restore step.** In a nested composite action the post step cannot see sibling step outputs, so a `path: ${{ steps.x.outputs.y }}` arrives empty there; the save then uses the paths restore resolved. + +Objects are stored as `/.cache.tzst`; authentication uses Application Default Credentials, so the credentials must still exist when the post step runs (put `google-github-actions/auth` **before** this action in the job, its cleanup runs in reverse order). + ### Cache scopes The cache is scoped to the key, [version](#cache-version), and branch. The default branch cache is available to other branches. diff --git a/__tests__/restore.test.ts b/__tests__/restore.test.ts index 250f7ef..c6432f4 100644 --- a/__tests__/restore.test.ts +++ b/__tests__/restore.test.ts @@ -85,7 +85,7 @@ test("restore with no cache found", async () => { ); expect(stateMock).toHaveBeenCalledWith("CACHE_KEY", key); - expect(stateMock).toHaveBeenCalledTimes(1); + expect(stateMock).toHaveBeenCalledTimes(2); expect(failedMock).toHaveBeenCalledTimes(0); @@ -128,7 +128,7 @@ test("restore with restore keys and no cache found", async () => { ); expect(stateMock).toHaveBeenCalledWith("CACHE_KEY", key); - expect(stateMock).toHaveBeenCalledTimes(1); + expect(stateMock).toHaveBeenCalledTimes(2); expect(failedMock).toHaveBeenCalledTimes(0); @@ -171,7 +171,7 @@ test("restore with cache found for key", async () => { expect(stateMock).toHaveBeenCalledWith("CACHE_KEY", key); expect(stateMock).toHaveBeenCalledWith("CACHE_RESULT", key); - expect(stateMock).toHaveBeenCalledTimes(2); + expect(stateMock).toHaveBeenCalledTimes(4); expect(setCacheHitOutputMock).toHaveBeenCalledTimes(1); expect(setCacheHitOutputMock).toHaveBeenCalledWith("cache-hit", "true"); @@ -216,7 +216,7 @@ test("restore with cache found for restore key", async () => { expect(stateMock).toHaveBeenCalledWith("CACHE_KEY", key); expect(stateMock).toHaveBeenCalledWith("CACHE_RESULT", restoreKey); - expect(stateMock).toHaveBeenCalledTimes(2); + expect(stateMock).toHaveBeenCalledTimes(4); expect(setCacheHitOutputMock).toHaveBeenCalledTimes(1); expect(setCacheHitOutputMock).toHaveBeenCalledWith("cache-hit", "false"); @@ -304,7 +304,7 @@ test("restore when fail on cache miss is enabled and primary key doesn't match r expect(stateMock).toHaveBeenCalledWith("CACHE_KEY", key); expect(stateMock).toHaveBeenCalledWith("CACHE_RESULT", restoreKey); - expect(stateMock).toHaveBeenCalledTimes(2); + expect(stateMock).toHaveBeenCalledTimes(4); expect(setCacheHitOutputMock).toHaveBeenCalledTimes(1); expect(setCacheHitOutputMock).toHaveBeenCalledWith("cache-hit", "false"); @@ -349,7 +349,7 @@ test("restore with fail on cache miss disabled and no cache found", async () => ); expect(stateMock).toHaveBeenCalledWith("CACHE_KEY", key); - expect(stateMock).toHaveBeenCalledTimes(1); + expect(stateMock).toHaveBeenCalledTimes(2); expect(infoMock).toHaveBeenCalledWith( `Cache not found for input keys: ${key}, ${restoreKey}` diff --git a/__tests__/restoreImpl.test.ts b/__tests__/restoreImpl.test.ts index 16f5f72..10b0c01 100644 --- a/__tests__/restoreImpl.test.ts +++ b/__tests__/restoreImpl.test.ts @@ -355,6 +355,9 @@ test("restore with cache found for key", async () => { ); expect(stateMock).toHaveBeenCalledWith("CACHE_KEY", key); + expect(stateMock).toHaveBeenCalledWith("CACHE_PATHS", path); + expect(stateMock).toHaveBeenCalledWith("CACHE_RESULT", key); + expect(stateMock).toHaveBeenCalledWith("CACHE_SOURCE", "github"); expect(setCacheHitOutputMock).toHaveBeenCalledTimes(1); expect(setCacheHitOutputMock).toHaveBeenCalledWith("cache-hit", "true"); @@ -439,7 +442,7 @@ test("restore with lookup-only set", async () => { expect(stateMock).toHaveBeenCalledWith("CACHE_KEY", key); expect(stateMock).toHaveBeenCalledWith("CACHE_RESULT", key); - expect(stateMock).toHaveBeenCalledTimes(2); + expect(stateMock).toHaveBeenCalledTimes(4); expect(setCacheHitOutputMock).toHaveBeenCalledTimes(1); expect(setCacheHitOutputMock).toHaveBeenCalledWith("cache-hit", "true"); diff --git a/__tests__/save.test.ts b/__tests__/save.test.ts index 4678c43..4e7db29 100644 --- a/__tests__/save.test.ts +++ b/__tests__/save.test.ts @@ -79,15 +79,11 @@ test("save with valid inputs uploads a cache", async () => { const primaryKey = "Linux-node-bb828da54c148048dd17899ba9fda624811cfb43"; const savedCacheKey = "Linux-node-"; - jest.spyOn(core, "getState") - // Cache Entry State - .mockImplementationOnce(() => { - return primaryKey; - }) - // Cache Key State - .mockImplementationOnce(() => { - return savedCacheKey; - }); + jest.spyOn(core, "getState").mockImplementation((name: string) => { + return ( + { CACHE_KEY: primaryKey, CACHE_RESULT: savedCacheKey }[name] ?? "" + ); + }); const inputPath = "node_modules"; testUtils.setInput(Inputs.Path, inputPath); diff --git a/__tests__/saveImpl.test.ts b/__tests__/saveImpl.test.ts index ad154f6..7e2ff30 100644 --- a/__tests__/saveImpl.test.ts +++ b/__tests__/saveImpl.test.ts @@ -5,12 +5,21 @@ import { Events, Inputs, RefKey } from "../src/constants"; import { saveImpl } from "../src/saveImpl"; import { StateProvider } from "../src/stateProvider"; import * as actionUtils from "../src/utils/actionUtils"; +import * as gcsCache from "../src/utils/gcsCache"; import * as testUtils from "../src/utils/testUtils"; jest.mock("@actions/core"); jest.mock("@actions/cache"); jest.mock("../src/utils/actionUtils"); +// The post step reads several states; key the mock by state name so a new +// read cannot shift which value the others receive. +function mockState(states: Record): void { + jest.spyOn(core, "getState").mockImplementation( + (name: string) => states[name] ?? "" + ); +} + beforeAll(() => { jest.spyOn(core, "getInput").mockImplementation((name, options) => { return jest.requireActual("@actions/core").getInput(name, options); @@ -89,15 +98,7 @@ test("save with no primary key in state outputs warning", async () => { const failedMock = jest.spyOn(core, "setFailed"); const savedCacheKey = "Linux-node-bb828da54c148048dd17899ba9fda624811cfb43"; - jest.spyOn(core, "getState") - // Cache Entry State - .mockImplementationOnce(() => { - return ""; - }) - // Cache Key State - .mockImplementationOnce(() => { - return savedCacheKey; - }); + mockState({ CACHE_KEY: "", CACHE_RESULT: savedCacheKey }); const saveCacheMock = jest.spyOn(cache, "saveCache"); await saveImpl(new StateProvider()); @@ -140,15 +141,7 @@ test("save on GHES with AC available", async () => { const primaryKey = "Linux-node-bb828da54c148048dd17899ba9fda624811cfb43"; const savedCacheKey = "Linux-node-"; - jest.spyOn(core, "getState") - // Cache Entry State - .mockImplementationOnce(() => { - return savedCacheKey; - }) - // Cache Key State - .mockImplementationOnce(() => { - return primaryKey; - }); + mockState({ CACHE_KEY: primaryKey, CACHE_RESULT: savedCacheKey }); const inputPath = "node_modules"; testUtils.setInput(Inputs.Path, inputPath); @@ -183,15 +176,7 @@ test("save with exact match returns early", async () => { const primaryKey = "Linux-node-bb828da54c148048dd17899ba9fda624811cfb43"; const savedCacheKey = primaryKey; - jest.spyOn(core, "getState") - // Cache Entry State - .mockImplementationOnce(() => { - return savedCacheKey; - }) - // Cache Key State - .mockImplementationOnce(() => { - return primaryKey; - }); + mockState({ CACHE_KEY: primaryKey, CACHE_RESULT: savedCacheKey }); const saveCacheMock = jest.spyOn(cache, "saveCache"); await saveImpl(new StateProvider()); @@ -210,15 +195,7 @@ test("save with missing input outputs warning", async () => { const primaryKey = "Linux-node-bb828da54c148048dd17899ba9fda624811cfb43"; const savedCacheKey = "Linux-node-"; - jest.spyOn(core, "getState") - // Cache Entry State - .mockImplementationOnce(() => { - return savedCacheKey; - }) - // Cache Key State - .mockImplementationOnce(() => { - return primaryKey; - }); + mockState({ CACHE_KEY: primaryKey, CACHE_RESULT: savedCacheKey }); const saveCacheMock = jest.spyOn(cache, "saveCache"); await saveImpl(new StateProvider()); @@ -238,15 +215,7 @@ test("save with large cache outputs warning", async () => { const primaryKey = "Linux-node-bb828da54c148048dd17899ba9fda624811cfb43"; const savedCacheKey = "Linux-node-"; - jest.spyOn(core, "getState") - // Cache Entry State - .mockImplementationOnce(() => { - return savedCacheKey; - }) - // Cache Key State - .mockImplementationOnce(() => { - return primaryKey; - }); + mockState({ CACHE_KEY: primaryKey, CACHE_RESULT: savedCacheKey }); const inputPath = "node_modules"; testUtils.setInput(Inputs.Path, inputPath); @@ -283,15 +252,7 @@ test("save with reserve cache failure outputs warning", async () => { const primaryKey = "Linux-node-bb828da54c148048dd17899ba9fda624811cfb43"; const savedCacheKey = "Linux-node-"; - jest.spyOn(core, "getState") - // Cache Entry State - .mockImplementationOnce(() => { - return savedCacheKey; - }) - // Cache Key State - .mockImplementationOnce(() => { - return primaryKey; - }); + mockState({ CACHE_KEY: primaryKey, CACHE_RESULT: savedCacheKey }); const inputPath = "node_modules"; testUtils.setInput(Inputs.Path, inputPath); @@ -330,15 +291,7 @@ test("save with server error outputs warning", async () => { const primaryKey = "Linux-node-bb828da54c148048dd17899ba9fda624811cfb43"; const savedCacheKey = "Linux-node-"; - jest.spyOn(core, "getState") - // Cache Entry State - .mockImplementationOnce(() => { - return savedCacheKey; - }) - // Cache Key State - .mockImplementationOnce(() => { - return primaryKey; - }); + mockState({ CACHE_KEY: primaryKey, CACHE_RESULT: savedCacheKey }); const inputPath = "node_modules"; testUtils.setInput(Inputs.Path, inputPath); @@ -371,15 +324,7 @@ test("save with valid inputs uploads a cache", async () => { const primaryKey = "Linux-node-bb828da54c148048dd17899ba9fda624811cfb43"; const savedCacheKey = "Linux-node-"; - jest.spyOn(core, "getState") - // Cache Entry State - .mockImplementationOnce(() => { - return savedCacheKey; - }) - // Cache Key State - .mockImplementationOnce(() => { - return primaryKey; - }); + mockState({ CACHE_KEY: primaryKey, CACHE_RESULT: savedCacheKey }); const inputPath = "node_modules"; testUtils.setInput(Inputs.Path, inputPath); @@ -406,3 +351,79 @@ test("save with valid inputs uploads a cache", async () => { expect(failedMock).toHaveBeenCalledTimes(0); }); + +test("save with exact match restored from GCS returns early", async () => { + const infoMock = jest.spyOn(core, "info"); + jest.spyOn(actionUtils, "isGCSAvailable").mockImplementation(() => true); + + const primaryKey = "Linux-node-bb828da54c148048dd17899ba9fda624811cfb43"; + mockState({ + CACHE_KEY: primaryKey, + CACHE_RESULT: primaryKey, + CACHE_SOURCE: "gcs" + }); + testUtils.setInput(Inputs.Path, "node_modules"); + const gcsSaveMock = jest.spyOn(gcsCache, "saveCache"); + + await saveImpl(new StateProvider()); + + expect(gcsSaveMock).toHaveBeenCalledTimes(0); + expect(infoMock).toHaveBeenCalledWith( + `Cache hit occurred on the primary key ${primaryKey}, not saving cache.` + ); +}); + +test("save with exact match restored from the GitHub cache backfills GCS", async () => { + const infoMock = jest.spyOn(core, "info"); + jest.spyOn(actionUtils, "isGCSAvailable").mockImplementation(() => true); + + const primaryKey = "Linux-node-bb828da54c148048dd17899ba9fda624811cfb43"; + mockState({ + CACHE_KEY: primaryKey, + CACHE_RESULT: primaryKey, + CACHE_SOURCE: "github" + }); + const inputPath = "node_modules"; + testUtils.setInput(Inputs.Path, inputPath); + testUtils.setInput(Inputs.UploadChunkSize, "4000000"); + const gcsSaveMock = jest + .spyOn(gcsCache, "saveCache") + .mockImplementation(() => Promise.resolve(1)); + + await saveImpl(new StateProvider()); + + expect(gcsSaveMock).toHaveBeenCalledTimes(1); + expect(gcsSaveMock).toHaveBeenCalledWith( + [inputPath], + primaryKey, + { uploadChunkSize: 4000000 }, + false, + false // GitHub already holds the entry: GCS only + ); + expect(infoMock).toHaveBeenCalledWith( + `Cache hit on the primary key ${primaryKey} came from the GitHub cache, saving it to GCS.` + ); +}); + +test("save with empty path input uses the paths recorded by restore", async () => { + const primaryKey = "Linux-node-bb828da54c148048dd17899ba9fda624811cfb43"; + mockState({ + CACHE_KEY: primaryKey, + CACHE_RESULT: "Linux-node-", + CACHE_PATHS: "node_modules\n~/.cache/Cypress" + }); + testUtils.setInput(Inputs.UploadChunkSize, "4000000"); + const saveCacheMock = jest + .spyOn(cache, "saveCache") + .mockImplementationOnce(() => Promise.resolve(4)); + + await saveImpl(new StateProvider()); + + expect(saveCacheMock).toHaveBeenCalledTimes(1); + expect(saveCacheMock).toHaveBeenCalledWith( + ["node_modules", "~/.cache/Cypress"], + primaryKey, + { uploadChunkSize: 4000000 }, + false + ); +}); diff --git a/dist/restore-only/index.js b/dist/restore-only/index.js index 3fb4e9b..5f6ac2d 100644 --- a/dist/restore-only/index.js +++ b/dist/restore-only/index.js @@ -78025,7 +78025,7 @@ module.exports = Queue; "use strict"; Object.defineProperty(exports, "__esModule", ({ value: true })); -exports.RefKey = exports.Events = exports.State = exports.Outputs = exports.Inputs = void 0; +exports.RefKey = exports.Events = exports.CacheSource = exports.State = exports.Outputs = exports.Inputs = void 0; var Inputs; (function (Inputs) { Inputs["Key"] = "key"; @@ -78048,7 +78048,19 @@ var State; (function (State) { State["CachePrimaryKey"] = "CACHE_KEY"; State["CacheMatchedKey"] = "CACHE_RESULT"; + // Which backend served the restore, so the post step knows whether a hit + // still has to be written to GCS. + State["CacheSource"] = "CACHE_SOURCE"; + // The resolved `path` input, newline-separated. A composite action's post + // step cannot see sibling step outputs, so `path: ${{ steps.x.outputs.y }}` + // arrives empty there; the save falls back to what restore saw. + State["CachePaths"] = "CACHE_PATHS"; })(State || (exports.State = State = {})); +var CacheSource; +(function (CacheSource) { + CacheSource["GCS"] = "gcs"; + CacheSource["GitHub"] = "github"; +})(CacheSource || (exports.CacheSource = CacheSource = {})); var Events; (function (Events) { Events["Key"] = "GITHUB_EVENT_NAME"; @@ -78134,11 +78146,13 @@ function restoreImpl(stateProvider, earlyExit) { const cachePaths = utils.getInputAsArray(constants_1.Inputs.Path, { required: true }); + stateProvider.setState(constants_1.State.CachePaths, cachePaths.join("\n")); const enableCrossOsArchive = utils.getInputAsBool(constants_1.Inputs.EnableCrossOsArchive); const failOnCacheMiss = utils.getInputAsBool(constants_1.Inputs.FailOnCacheMiss); const lookupOnly = utils.getInputAsBool(constants_1.Inputs.LookupOnly); - const cacheKey = yield cache.restoreCache(cachePaths, primaryKey, restoreKeys, { lookupOnly: lookupOnly }, enableCrossOsArchive); - if (!cacheKey) { + const restored = yield cache.restoreCache(cachePaths, primaryKey, restoreKeys, { lookupOnly: lookupOnly }, enableCrossOsArchive); + const cacheKey = restored === null || restored === void 0 ? void 0 : restored.key; + if (!restored || !cacheKey) { // `cache-hit` is intentionally not set to `false` here to preserve existing behavior // See https://github.com/actions/cache/issues/1466 if (failOnCacheMiss) { @@ -78150,8 +78164,9 @@ function restoreImpl(stateProvider, earlyExit) { ].join(", ")}`); return; } - // Store the matched cache key in states + // Store the matched cache key and its backend in states stateProvider.setState(constants_1.State.CacheMatchedKey, cacheKey); + stateProvider.setState(constants_1.State.CacheSource, restored.source); const isExactKeyMatch = utils.isExactKeyMatch(core.getInput(constants_1.Inputs.Key, { required: true }), cacheKey); core.setOutput(constants_1.Outputs.CacheHit, isExactKeyMatch.toString()); if (lookupOnly) { @@ -78270,8 +78285,13 @@ class NullStateProvider extends StateProviderBase { [constants_1.State.CacheMatchedKey, constants_1.Outputs.CacheMatchedKey], [constants_1.State.CachePrimaryKey, constants_1.Outputs.CachePrimaryKey] ]); + // Only states with an output counterpart are exposed; the rest are + // save-step bookkeeping that a restore-only action has no post step for. this.setState = (key, value) => { - core.setOutput(this.stateToOutputMap.get(key), value); + const output = this.stateToOutputMap.get(key); + if (output) { + core.setOutput(output, value); + } }; // eslint-disable-next-line @typescript-eslint/no-unused-vars this.getState = (key) => ""; @@ -78504,7 +78524,7 @@ function restoreCache(paths, primaryKey, restoreKeys, options, enableCrossOsArch const result = yield restoreFromGCS(paths, primaryKey, restoreKeys, options); if (result) { core.info(`Cache restored from GCS with key: ${result}`); - return result; + return { key: result, source: constants_1.CacheSource.GCS }; } core.info("Cache not found in GCS, falling back to GitHub cache"); } @@ -78517,11 +78537,18 @@ function restoreCache(paths, primaryKey, restoreKeys, options, enableCrossOsArch core.info("GCS not configured, using GitHub cache"); } // Fall back to GitHub cache - return yield cache.restoreCache(paths, primaryKey, restoreKeys, options, enableCrossOsArchive); + const key = yield cache.restoreCache(paths, primaryKey, restoreKeys, options, enableCrossOsArchive); + return key ? { key, source: constants_1.CacheSource.GitHub } : undefined; }); } -function saveCache(paths, key, options, enableCrossOsArchive) { - return __awaiter(this, void 0, void 0, function* () { +/** + * Saves to GCS when it is configured, otherwise (or when the GCS upload + * fails) to the GitHub cache. `fallbackToGitHub: false` is for backfilling a + * GCS miss the GitHub cache already covered: a second GitHub save would only + * fail on the existing entry. + */ +function saveCache(paths_1, key_1, options_1, enableCrossOsArchive_1) { + return __awaiter(this, arguments, void 0, function* (paths, key, options, enableCrossOsArchive, fallbackToGitHub = true) { if ((0, actionUtils_1.isGCSAvailable)()) { try { const result = yield saveToGCS(paths, key); @@ -78529,15 +78556,20 @@ function saveCache(paths, key, options, enableCrossOsArchive) { core.info(`Cache saved to GCS with key: [${key} | ${result}]`); return 1; // Success ID } - core.warning("Failed to save to GCS, falling back to GitHub cache"); - return -1; + core.warning("Failed to save to GCS"); } catch (error) { core.warning(`Failed to save to GCS: ${error.message}`); - core.info("Falling back to GitHub cache"); } + if (!fallbackToGitHub) { + return -1; + } + core.info("Falling back to GitHub cache"); } else { + if (!fallbackToGitHub) { + return -1; + } core.info("GCS not configured, using GitHub cache"); } // Fall back to GitHub cache diff --git a/dist/restore/index.js b/dist/restore/index.js index d436f8d..2707714 100644 --- a/dist/restore/index.js +++ b/dist/restore/index.js @@ -78025,7 +78025,7 @@ module.exports = Queue; "use strict"; Object.defineProperty(exports, "__esModule", ({ value: true })); -exports.RefKey = exports.Events = exports.State = exports.Outputs = exports.Inputs = void 0; +exports.RefKey = exports.Events = exports.CacheSource = exports.State = exports.Outputs = exports.Inputs = void 0; var Inputs; (function (Inputs) { Inputs["Key"] = "key"; @@ -78048,7 +78048,19 @@ var State; (function (State) { State["CachePrimaryKey"] = "CACHE_KEY"; State["CacheMatchedKey"] = "CACHE_RESULT"; + // Which backend served the restore, so the post step knows whether a hit + // still has to be written to GCS. + State["CacheSource"] = "CACHE_SOURCE"; + // The resolved `path` input, newline-separated. A composite action's post + // step cannot see sibling step outputs, so `path: ${{ steps.x.outputs.y }}` + // arrives empty there; the save falls back to what restore saw. + State["CachePaths"] = "CACHE_PATHS"; })(State || (exports.State = State = {})); +var CacheSource; +(function (CacheSource) { + CacheSource["GCS"] = "gcs"; + CacheSource["GitHub"] = "github"; +})(CacheSource || (exports.CacheSource = CacheSource = {})); var Events; (function (Events) { Events["Key"] = "GITHUB_EVENT_NAME"; @@ -78134,11 +78146,13 @@ function restoreImpl(stateProvider, earlyExit) { const cachePaths = utils.getInputAsArray(constants_1.Inputs.Path, { required: true }); + stateProvider.setState(constants_1.State.CachePaths, cachePaths.join("\n")); const enableCrossOsArchive = utils.getInputAsBool(constants_1.Inputs.EnableCrossOsArchive); const failOnCacheMiss = utils.getInputAsBool(constants_1.Inputs.FailOnCacheMiss); const lookupOnly = utils.getInputAsBool(constants_1.Inputs.LookupOnly); - const cacheKey = yield cache.restoreCache(cachePaths, primaryKey, restoreKeys, { lookupOnly: lookupOnly }, enableCrossOsArchive); - if (!cacheKey) { + const restored = yield cache.restoreCache(cachePaths, primaryKey, restoreKeys, { lookupOnly: lookupOnly }, enableCrossOsArchive); + const cacheKey = restored === null || restored === void 0 ? void 0 : restored.key; + if (!restored || !cacheKey) { // `cache-hit` is intentionally not set to `false` here to preserve existing behavior // See https://github.com/actions/cache/issues/1466 if (failOnCacheMiss) { @@ -78150,8 +78164,9 @@ function restoreImpl(stateProvider, earlyExit) { ].join(", ")}`); return; } - // Store the matched cache key in states + // Store the matched cache key and its backend in states stateProvider.setState(constants_1.State.CacheMatchedKey, cacheKey); + stateProvider.setState(constants_1.State.CacheSource, restored.source); const isExactKeyMatch = utils.isExactKeyMatch(core.getInput(constants_1.Inputs.Key, { required: true }), cacheKey); core.setOutput(constants_1.Outputs.CacheHit, isExactKeyMatch.toString()); if (lookupOnly) { @@ -78270,8 +78285,13 @@ class NullStateProvider extends StateProviderBase { [constants_1.State.CacheMatchedKey, constants_1.Outputs.CacheMatchedKey], [constants_1.State.CachePrimaryKey, constants_1.Outputs.CachePrimaryKey] ]); + // Only states with an output counterpart are exposed; the rest are + // save-step bookkeeping that a restore-only action has no post step for. this.setState = (key, value) => { - core.setOutput(this.stateToOutputMap.get(key), value); + const output = this.stateToOutputMap.get(key); + if (output) { + core.setOutput(output, value); + } }; // eslint-disable-next-line @typescript-eslint/no-unused-vars this.getState = (key) => ""; @@ -78504,7 +78524,7 @@ function restoreCache(paths, primaryKey, restoreKeys, options, enableCrossOsArch const result = yield restoreFromGCS(paths, primaryKey, restoreKeys, options); if (result) { core.info(`Cache restored from GCS with key: ${result}`); - return result; + return { key: result, source: constants_1.CacheSource.GCS }; } core.info("Cache not found in GCS, falling back to GitHub cache"); } @@ -78517,11 +78537,18 @@ function restoreCache(paths, primaryKey, restoreKeys, options, enableCrossOsArch core.info("GCS not configured, using GitHub cache"); } // Fall back to GitHub cache - return yield cache.restoreCache(paths, primaryKey, restoreKeys, options, enableCrossOsArchive); + const key = yield cache.restoreCache(paths, primaryKey, restoreKeys, options, enableCrossOsArchive); + return key ? { key, source: constants_1.CacheSource.GitHub } : undefined; }); } -function saveCache(paths, key, options, enableCrossOsArchive) { - return __awaiter(this, void 0, void 0, function* () { +/** + * Saves to GCS when it is configured, otherwise (or when the GCS upload + * fails) to the GitHub cache. `fallbackToGitHub: false` is for backfilling a + * GCS miss the GitHub cache already covered: a second GitHub save would only + * fail on the existing entry. + */ +function saveCache(paths_1, key_1, options_1, enableCrossOsArchive_1) { + return __awaiter(this, arguments, void 0, function* (paths, key, options, enableCrossOsArchive, fallbackToGitHub = true) { if ((0, actionUtils_1.isGCSAvailable)()) { try { const result = yield saveToGCS(paths, key); @@ -78529,15 +78556,20 @@ function saveCache(paths, key, options, enableCrossOsArchive) { core.info(`Cache saved to GCS with key: [${key} | ${result}]`); return 1; // Success ID } - core.warning("Failed to save to GCS, falling back to GitHub cache"); - return -1; + core.warning("Failed to save to GCS"); } catch (error) { core.warning(`Failed to save to GCS: ${error.message}`); - core.info("Falling back to GitHub cache"); } + if (!fallbackToGitHub) { + return -1; + } + core.info("Falling back to GitHub cache"); } else { + if (!fallbackToGitHub) { + return -1; + } core.info("GCS not configured, using GitHub cache"); } // Fall back to GitHub cache diff --git a/dist/save-only/index.js b/dist/save-only/index.js index e64bfb9..48b21ec 100644 --- a/dist/save-only/index.js +++ b/dist/save-only/index.js @@ -78025,7 +78025,7 @@ module.exports = Queue; "use strict"; Object.defineProperty(exports, "__esModule", ({ value: true })); -exports.RefKey = exports.Events = exports.State = exports.Outputs = exports.Inputs = void 0; +exports.RefKey = exports.Events = exports.CacheSource = exports.State = exports.Outputs = exports.Inputs = void 0; var Inputs; (function (Inputs) { Inputs["Key"] = "key"; @@ -78048,7 +78048,19 @@ var State; (function (State) { State["CachePrimaryKey"] = "CACHE_KEY"; State["CacheMatchedKey"] = "CACHE_RESULT"; + // Which backend served the restore, so the post step knows whether a hit + // still has to be written to GCS. + State["CacheSource"] = "CACHE_SOURCE"; + // The resolved `path` input, newline-separated. A composite action's post + // step cannot see sibling step outputs, so `path: ${{ steps.x.outputs.y }}` + // arrives empty there; the save falls back to what restore saw. + State["CachePaths"] = "CACHE_PATHS"; })(State || (exports.State = State = {})); +var CacheSource; +(function (CacheSource) { + CacheSource["GCS"] = "gcs"; + CacheSource["GitHub"] = "github"; +})(CacheSource || (exports.CacheSource = CacheSource = {})); var Events; (function (Events) { Events["Key"] = "GITHUB_EVENT_NAME"; @@ -78141,16 +78153,37 @@ function saveImpl(stateProvider) { } // If matched restore key is same as primary key, then do not save cache // NO-OP in case of SaveOnly action + // + // Exception: a hit served by the GitHub fallback while GCS is + // configured. GCS is the primary backend, so the entry is written + // there too — otherwise the GitHub copy keeps every later job on the + // fallback and GCS never gets the key. const restoredKey = stateProvider.getCacheState(); - if (utils.isExactKeyMatch(primaryKey, restoredKey)) { + const exactMatch = utils.isExactKeyMatch(primaryKey, restoredKey); + const backfillGCS = exactMatch && + stateProvider.getState(constants_1.State.CacheSource) === constants_1.CacheSource.GitHub && + utils.isGCSAvailable(); + if (exactMatch && !backfillGCS) { core.info(`Cache hit occurred on the primary key ${primaryKey}, not saving cache.`); return; } - const cachePaths = utils.getInputAsArray(constants_1.Inputs.Path, { - required: true - }); + if (backfillGCS) { + core.info(`Cache hit on the primary key ${primaryKey} came from the GitHub cache, saving it to GCS.`); + } + // Prefer the paths restore recorded: in a nested composite action the + // `path` input is empty in the post step (see State.CachePaths). + const inputPaths = utils.getInputAsArray(constants_1.Inputs.Path); + const cachePaths = inputPaths.length + ? inputPaths + : (stateProvider.getState(constants_1.State.CachePaths) || "") + .split("\n") + .filter(Boolean); + if (cachePaths.length === 0) { + utils.logWarning("Input required and not supplied: path"); + return; + } const enableCrossOsArchive = utils.getInputAsBool(constants_1.Inputs.EnableCrossOsArchive); - cacheId = yield cache.saveCache(cachePaths, primaryKey, { uploadChunkSize: utils.getInputAsInt(constants_1.Inputs.UploadChunkSize) }, enableCrossOsArchive); + cacheId = yield cache.saveCache(cachePaths, primaryKey, { uploadChunkSize: utils.getInputAsInt(constants_1.Inputs.UploadChunkSize) }, enableCrossOsArchive, !backfillGCS); if (cacheId != -1) { core.info(`Cache saved with key: ${primaryKey}`); } @@ -78283,8 +78316,13 @@ class NullStateProvider extends StateProviderBase { [constants_1.State.CacheMatchedKey, constants_1.Outputs.CacheMatchedKey], [constants_1.State.CachePrimaryKey, constants_1.Outputs.CachePrimaryKey] ]); + // Only states with an output counterpart are exposed; the rest are + // save-step bookkeeping that a restore-only action has no post step for. this.setState = (key, value) => { - core.setOutput(this.stateToOutputMap.get(key), value); + const output = this.stateToOutputMap.get(key); + if (output) { + core.setOutput(output, value); + } }; // eslint-disable-next-line @typescript-eslint/no-unused-vars this.getState = (key) => ""; @@ -78517,7 +78555,7 @@ function restoreCache(paths, primaryKey, restoreKeys, options, enableCrossOsArch const result = yield restoreFromGCS(paths, primaryKey, restoreKeys, options); if (result) { core.info(`Cache restored from GCS with key: ${result}`); - return result; + return { key: result, source: constants_1.CacheSource.GCS }; } core.info("Cache not found in GCS, falling back to GitHub cache"); } @@ -78530,11 +78568,18 @@ function restoreCache(paths, primaryKey, restoreKeys, options, enableCrossOsArch core.info("GCS not configured, using GitHub cache"); } // Fall back to GitHub cache - return yield cache.restoreCache(paths, primaryKey, restoreKeys, options, enableCrossOsArchive); + const key = yield cache.restoreCache(paths, primaryKey, restoreKeys, options, enableCrossOsArchive); + return key ? { key, source: constants_1.CacheSource.GitHub } : undefined; }); } -function saveCache(paths, key, options, enableCrossOsArchive) { - return __awaiter(this, void 0, void 0, function* () { +/** + * Saves to GCS when it is configured, otherwise (or when the GCS upload + * fails) to the GitHub cache. `fallbackToGitHub: false` is for backfilling a + * GCS miss the GitHub cache already covered: a second GitHub save would only + * fail on the existing entry. + */ +function saveCache(paths_1, key_1, options_1, enableCrossOsArchive_1) { + return __awaiter(this, arguments, void 0, function* (paths, key, options, enableCrossOsArchive, fallbackToGitHub = true) { if ((0, actionUtils_1.isGCSAvailable)()) { try { const result = yield saveToGCS(paths, key); @@ -78542,15 +78587,20 @@ function saveCache(paths, key, options, enableCrossOsArchive) { core.info(`Cache saved to GCS with key: [${key} | ${result}]`); return 1; // Success ID } - core.warning("Failed to save to GCS, falling back to GitHub cache"); - return -1; + core.warning("Failed to save to GCS"); } catch (error) { core.warning(`Failed to save to GCS: ${error.message}`); - core.info("Falling back to GitHub cache"); } + if (!fallbackToGitHub) { + return -1; + } + core.info("Falling back to GitHub cache"); } else { + if (!fallbackToGitHub) { + return -1; + } core.info("GCS not configured, using GitHub cache"); } // Fall back to GitHub cache diff --git a/dist/save/index.js b/dist/save/index.js index 6237269..ed7127b 100644 --- a/dist/save/index.js +++ b/dist/save/index.js @@ -78025,7 +78025,7 @@ module.exports = Queue; "use strict"; Object.defineProperty(exports, "__esModule", ({ value: true })); -exports.RefKey = exports.Events = exports.State = exports.Outputs = exports.Inputs = void 0; +exports.RefKey = exports.Events = exports.CacheSource = exports.State = exports.Outputs = exports.Inputs = void 0; var Inputs; (function (Inputs) { Inputs["Key"] = "key"; @@ -78048,7 +78048,19 @@ var State; (function (State) { State["CachePrimaryKey"] = "CACHE_KEY"; State["CacheMatchedKey"] = "CACHE_RESULT"; + // Which backend served the restore, so the post step knows whether a hit + // still has to be written to GCS. + State["CacheSource"] = "CACHE_SOURCE"; + // The resolved `path` input, newline-separated. A composite action's post + // step cannot see sibling step outputs, so `path: ${{ steps.x.outputs.y }}` + // arrives empty there; the save falls back to what restore saw. + State["CachePaths"] = "CACHE_PATHS"; })(State || (exports.State = State = {})); +var CacheSource; +(function (CacheSource) { + CacheSource["GCS"] = "gcs"; + CacheSource["GitHub"] = "github"; +})(CacheSource || (exports.CacheSource = CacheSource = {})); var Events; (function (Events) { Events["Key"] = "GITHUB_EVENT_NAME"; @@ -78141,16 +78153,37 @@ function saveImpl(stateProvider) { } // If matched restore key is same as primary key, then do not save cache // NO-OP in case of SaveOnly action + // + // Exception: a hit served by the GitHub fallback while GCS is + // configured. GCS is the primary backend, so the entry is written + // there too — otherwise the GitHub copy keeps every later job on the + // fallback and GCS never gets the key. const restoredKey = stateProvider.getCacheState(); - if (utils.isExactKeyMatch(primaryKey, restoredKey)) { + const exactMatch = utils.isExactKeyMatch(primaryKey, restoredKey); + const backfillGCS = exactMatch && + stateProvider.getState(constants_1.State.CacheSource) === constants_1.CacheSource.GitHub && + utils.isGCSAvailable(); + if (exactMatch && !backfillGCS) { core.info(`Cache hit occurred on the primary key ${primaryKey}, not saving cache.`); return; } - const cachePaths = utils.getInputAsArray(constants_1.Inputs.Path, { - required: true - }); + if (backfillGCS) { + core.info(`Cache hit on the primary key ${primaryKey} came from the GitHub cache, saving it to GCS.`); + } + // Prefer the paths restore recorded: in a nested composite action the + // `path` input is empty in the post step (see State.CachePaths). + const inputPaths = utils.getInputAsArray(constants_1.Inputs.Path); + const cachePaths = inputPaths.length + ? inputPaths + : (stateProvider.getState(constants_1.State.CachePaths) || "") + .split("\n") + .filter(Boolean); + if (cachePaths.length === 0) { + utils.logWarning("Input required and not supplied: path"); + return; + } const enableCrossOsArchive = utils.getInputAsBool(constants_1.Inputs.EnableCrossOsArchive); - cacheId = yield cache.saveCache(cachePaths, primaryKey, { uploadChunkSize: utils.getInputAsInt(constants_1.Inputs.UploadChunkSize) }, enableCrossOsArchive); + cacheId = yield cache.saveCache(cachePaths, primaryKey, { uploadChunkSize: utils.getInputAsInt(constants_1.Inputs.UploadChunkSize) }, enableCrossOsArchive, !backfillGCS); if (cacheId != -1) { core.info(`Cache saved with key: ${primaryKey}`); } @@ -78283,8 +78316,13 @@ class NullStateProvider extends StateProviderBase { [constants_1.State.CacheMatchedKey, constants_1.Outputs.CacheMatchedKey], [constants_1.State.CachePrimaryKey, constants_1.Outputs.CachePrimaryKey] ]); + // Only states with an output counterpart are exposed; the rest are + // save-step bookkeeping that a restore-only action has no post step for. this.setState = (key, value) => { - core.setOutput(this.stateToOutputMap.get(key), value); + const output = this.stateToOutputMap.get(key); + if (output) { + core.setOutput(output, value); + } }; // eslint-disable-next-line @typescript-eslint/no-unused-vars this.getState = (key) => ""; @@ -78517,7 +78555,7 @@ function restoreCache(paths, primaryKey, restoreKeys, options, enableCrossOsArch const result = yield restoreFromGCS(paths, primaryKey, restoreKeys, options); if (result) { core.info(`Cache restored from GCS with key: ${result}`); - return result; + return { key: result, source: constants_1.CacheSource.GCS }; } core.info("Cache not found in GCS, falling back to GitHub cache"); } @@ -78530,11 +78568,18 @@ function restoreCache(paths, primaryKey, restoreKeys, options, enableCrossOsArch core.info("GCS not configured, using GitHub cache"); } // Fall back to GitHub cache - return yield cache.restoreCache(paths, primaryKey, restoreKeys, options, enableCrossOsArchive); + const key = yield cache.restoreCache(paths, primaryKey, restoreKeys, options, enableCrossOsArchive); + return key ? { key, source: constants_1.CacheSource.GitHub } : undefined; }); } -function saveCache(paths, key, options, enableCrossOsArchive) { - return __awaiter(this, void 0, void 0, function* () { +/** + * Saves to GCS when it is configured, otherwise (or when the GCS upload + * fails) to the GitHub cache. `fallbackToGitHub: false` is for backfilling a + * GCS miss the GitHub cache already covered: a second GitHub save would only + * fail on the existing entry. + */ +function saveCache(paths_1, key_1, options_1, enableCrossOsArchive_1) { + return __awaiter(this, arguments, void 0, function* (paths, key, options, enableCrossOsArchive, fallbackToGitHub = true) { if ((0, actionUtils_1.isGCSAvailable)()) { try { const result = yield saveToGCS(paths, key); @@ -78542,15 +78587,20 @@ function saveCache(paths, key, options, enableCrossOsArchive) { core.info(`Cache saved to GCS with key: [${key} | ${result}]`); return 1; // Success ID } - core.warning("Failed to save to GCS, falling back to GitHub cache"); - return -1; + core.warning("Failed to save to GCS"); } catch (error) { core.warning(`Failed to save to GCS: ${error.message}`); - core.info("Falling back to GitHub cache"); } + if (!fallbackToGitHub) { + return -1; + } + core.info("Falling back to GitHub cache"); } else { + if (!fallbackToGitHub) { + return -1; + } core.info("GCS not configured, using GitHub cache"); } // Fall back to GitHub cache diff --git a/src/constants.ts b/src/constants.ts index 13173db..8caa286 100644 --- a/src/constants.ts +++ b/src/constants.ts @@ -18,7 +18,19 @@ export enum Outputs { export enum State { CachePrimaryKey = "CACHE_KEY", - CacheMatchedKey = "CACHE_RESULT" + CacheMatchedKey = "CACHE_RESULT", + // Which backend served the restore, so the post step knows whether a hit + // still has to be written to GCS. + CacheSource = "CACHE_SOURCE", + // The resolved `path` input, newline-separated. A composite action's post + // step cannot see sibling step outputs, so `path: ${{ steps.x.outputs.y }}` + // arrives empty there; the save falls back to what restore saw. + CachePaths = "CACHE_PATHS" +} + +export enum CacheSource { + GCS = "gcs", + GitHub = "github" } export enum Events { diff --git a/src/restoreImpl.ts b/src/restoreImpl.ts index 598ac63..a8ba964 100644 --- a/src/restoreImpl.ts +++ b/src/restoreImpl.ts @@ -36,21 +36,23 @@ export async function restoreImpl( const cachePaths = utils.getInputAsArray(Inputs.Path, { required: true }); + stateProvider.setState(State.CachePaths, cachePaths.join("\n")); const enableCrossOsArchive = utils.getInputAsBool( Inputs.EnableCrossOsArchive ); const failOnCacheMiss = utils.getInputAsBool(Inputs.FailOnCacheMiss); const lookupOnly = utils.getInputAsBool(Inputs.LookupOnly); - const cacheKey = await cache.restoreCache( + const restored = await cache.restoreCache( cachePaths, primaryKey, restoreKeys, { lookupOnly: lookupOnly }, enableCrossOsArchive ); + const cacheKey = restored?.key; - if (!cacheKey) { + if (!restored || !cacheKey) { // `cache-hit` is intentionally not set to `false` here to preserve existing behavior // See https://github.com/actions/cache/issues/1466 @@ -68,8 +70,9 @@ export async function restoreImpl( return; } - // Store the matched cache key in states + // Store the matched cache key and its backend in states stateProvider.setState(State.CacheMatchedKey, cacheKey); + stateProvider.setState(State.CacheSource, restored.source); const isExactKeyMatch = utils.isExactKeyMatch( core.getInput(Inputs.Key, { required: true }), diff --git a/src/saveImpl.ts b/src/saveImpl.ts index 1abf710..dae7233 100644 --- a/src/saveImpl.ts +++ b/src/saveImpl.ts @@ -1,6 +1,6 @@ import * as core from "@actions/core"; -import { Events, Inputs, State } from "./constants"; +import { CacheSource, Events, Inputs, State } from "./constants"; import { IStateProvider, NullStateProvider, @@ -45,18 +45,42 @@ export async function saveImpl( // If matched restore key is same as primary key, then do not save cache // NO-OP in case of SaveOnly action + // + // Exception: a hit served by the GitHub fallback while GCS is + // configured. GCS is the primary backend, so the entry is written + // there too — otherwise the GitHub copy keeps every later job on the + // fallback and GCS never gets the key. const restoredKey = stateProvider.getCacheState(); + const exactMatch = utils.isExactKeyMatch(primaryKey, restoredKey); + const backfillGCS = + exactMatch && + stateProvider.getState(State.CacheSource) === CacheSource.GitHub && + utils.isGCSAvailable(); - if (utils.isExactKeyMatch(primaryKey, restoredKey)) { + if (exactMatch && !backfillGCS) { core.info( `Cache hit occurred on the primary key ${primaryKey}, not saving cache.` ); return; } + if (backfillGCS) { + core.info( + `Cache hit on the primary key ${primaryKey} came from the GitHub cache, saving it to GCS.` + ); + } - const cachePaths = utils.getInputAsArray(Inputs.Path, { - required: true - }); + // Prefer the paths restore recorded: in a nested composite action the + // `path` input is empty in the post step (see State.CachePaths). + const inputPaths = utils.getInputAsArray(Inputs.Path); + const cachePaths = inputPaths.length + ? inputPaths + : (stateProvider.getState(State.CachePaths) || "") + .split("\n") + .filter(Boolean); + if (cachePaths.length === 0) { + utils.logWarning("Input required and not supplied: path"); + return; + } const enableCrossOsArchive = utils.getInputAsBool( Inputs.EnableCrossOsArchive @@ -66,7 +90,8 @@ export async function saveImpl( cachePaths, primaryKey, { uploadChunkSize: utils.getInputAsInt(Inputs.UploadChunkSize) }, - enableCrossOsArchive + enableCrossOsArchive, + !backfillGCS ); if (cacheId != -1) { diff --git a/src/stateProvider.ts b/src/stateProvider.ts index beb41e5..5a21015 100644 --- a/src/stateProvider.ts +++ b/src/stateProvider.ts @@ -38,8 +38,13 @@ export class NullStateProvider extends StateProviderBase { [State.CachePrimaryKey, Outputs.CachePrimaryKey] ]); + // Only states with an output counterpart are exposed; the rest are + // save-step bookkeeping that a restore-only action has no post step for. setState = (key: string, value: string) => { - core.setOutput(this.stateToOutputMap.get(key) as string, value); + const output = this.stateToOutputMap.get(key); + if (output) { + core.setOutput(output, value); + } }; // eslint-disable-next-line @typescript-eslint/no-unused-vars getState = (key: string) => ""; diff --git a/src/utils/gcsCache.ts b/src/utils/gcsCache.ts index c1ffd3e..906ebac 100644 --- a/src/utils/gcsCache.ts +++ b/src/utils/gcsCache.ts @@ -11,11 +11,16 @@ import * as core from "@actions/core"; import { Storage } from "@google-cloud/storage"; import * as path from "path"; -import { Inputs } from "../constants"; +import { CacheSource, Inputs } from "../constants"; import { getGCSBucket, isGCSAvailable } from "./actionUtils"; const DEFAULT_PATH_PREFIX = "github-cache"; +export interface RestoreResult { + key: string; + source: CacheSource; +} + // Function to initialize GCS client using Application Default Credentials function getGCSClient(): Storage | null { try { @@ -35,7 +40,7 @@ export async function restoreCache( restoreKeys?: string[], options?: DownloadOptions, enableCrossOsArchive?: boolean -): Promise { +): Promise { // Check if GCS is available if (isGCSAvailable()) { try { @@ -48,7 +53,7 @@ export async function restoreCache( if (result) { core.info(`Cache restored from GCS with key: ${result}`); - return result; + return { key: result, source: CacheSource.GCS }; } core.info("Cache not found in GCS, falling back to GitHub cache"); @@ -63,20 +68,28 @@ export async function restoreCache( } // Fall back to GitHub cache - return await cache.restoreCache( + const key = await cache.restoreCache( paths, primaryKey, restoreKeys, options, enableCrossOsArchive ); + return key ? { key, source: CacheSource.GitHub } : undefined; } +/** + * Saves to GCS when it is configured, otherwise (or when the GCS upload + * fails) to the GitHub cache. `fallbackToGitHub: false` is for backfilling a + * GCS miss the GitHub cache already covered: a second GitHub save would only + * fail on the existing entry. + */ export async function saveCache( paths: string[], key: string, options?: UploadOptions, - enableCrossOsArchive?: boolean + enableCrossOsArchive?: boolean, + fallbackToGitHub = true ): Promise { if (isGCSAvailable()) { try { @@ -86,13 +99,18 @@ export async function saveCache( return 1; // Success ID } - core.warning("Failed to save to GCS, falling back to GitHub cache"); - return -1; + core.warning("Failed to save to GCS"); } catch (error) { core.warning(`Failed to save to GCS: ${(error as Error).message}`); - core.info("Falling back to GitHub cache"); } + if (!fallbackToGitHub) { + return -1; + } + core.info("Falling back to GitHub cache"); } else { + if (!fallbackToGitHub) { + return -1; + } core.info("GCS not configured, using GitHub cache"); }