gcs used for backing

README.md

@@ -1,3 +1,5 @@
+# **Disclaimer**: Modified to use GCS bucket as backing. Only supports default cache action and assumes gsutil is configured as well as gcloud auth.
+
 # Cache action
 
 This action allows caching dependencies and build outputs to improve workflow execution time.

action.yml

@@ -2,6 +2,10 @@ name: 'Cache'
 description: 'Cache artifacts like dependencies and build outputs to improve workflow execution time'
 author: 'GitHub'
 inputs:
+  bucket:
+    description: 'The GCS bucket holding cache.'
+    default: 'ingestion-engine-dev-scratch-artifacts'
+    required: false
   path:
     description: 'A list of files, directories, and wildcard patterns to cache and restore'
     required: true
@@ -9,21 +13,21 @@ inputs:
     description: 'An explicit key for restoring and saving the cache'
     required: true
   restore-keys:
-    description: 'An ordered list of keys to use for restoring stale cache if no cache hit occurred for key. Note `cache-hit` returns false in this case.'
+    description: 'UNIMPLEMENTED: An ordered list of keys to use for restoring stale cache if no cache hit occurred for key. Note `cache-hit` returns false in this case.'
     required: false
   upload-chunk-size:
-    description: 'The chunk size used to split up large files during upload, in bytes'
+    description: 'UNIMPLEMENTED: The chunk size used to split up large files during upload, in bytes'
     required: false
   enableCrossOsArchive:
-    description: 'An optional boolean when enabled, allows windows runners to save or restore caches that can be restored or saved respectively on other platforms'
+    description: 'UNIMPLEMENTED: An optional boolean when enabled, allows windows runners to save or restore caches that can be restored or saved respectively on other platforms'
     default: 'false'
     required: false
   fail-on-cache-miss:
-    description: 'Fail the workflow if cache entry is not found'
+    description: 'UNIMPLEMENTED: Fail the workflow if cache entry is not found'
     default: 'false'
     required: false
   lookup-only:
-    description: 'Check if a cache entry exists for the given input(s) (key, restore-keys) without downloading the cache'
+    description: 'UNIMPLEMENTED: Check if a cache entry exists for the given input(s) (key, restore-keys) without downloading the cache'
     default: 'false'
     required: false
 outputs:

src/constants.ts

@@ -1,4 +1,5 @@
 export enum Inputs {
+    Bucket = "bucket", // Input for cache action.
     Key = "key", // Input for cache, restore, save action
     Path = "path", // Input for cache, restore, save action
     RestoreKeys = "restore-keys", // Input for cache, restore action

src/restoreImpl.ts

@@ -1,17 +1,10 @@
-import * as cache from "@actions/cache";
 import * as core from "@actions/core";
+import { exec } from "@actions/exec";
 
-import { Events, Inputs, Outputs, State } from "./constants";
-import {
-    IStateProvider,
-    NullStateProvider,
-    StateProvider
-} from "./stateProvider";
+import { Events, Inputs, Outputs } from "./constants";
 import * as utils from "./utils/actionUtils";
 
-export async function restoreImpl(
-    stateProvider: IStateProvider
-): Promise<string | undefined> {
+export async function restoreImpl(): Promise<string | undefined> {
     try {
         if (!utils.isCacheFeatureAvailable()) {
             core.setOutput(Outputs.CacheHit, "false");
@@ -28,70 +21,28 @@ export async function restoreImpl(
             return;
         }
 
-        const primaryKey = core.getInput(Inputs.Key, { required: true });
-        stateProvider.setState(State.CachePrimaryKey, primaryKey);
-
-        const restoreKeys = utils.getInputAsArray(Inputs.RestoreKeys);
-        const cachePaths = utils.getInputAsArray(Inputs.Path, {
-            required: true
-        });
-        const enableCrossOsArchive = utils.getInputAsBool(
-            Inputs.EnableCrossOsArchive
-        );
-        const failOnCacheMiss = utils.getInputAsBool(Inputs.FailOnCacheMiss);
-        const lookupOnly = utils.getInputAsBool(Inputs.LookupOnly);
-
-        const cacheKey = await cache.restoreCache(
-            cachePaths,
-            primaryKey,
-            restoreKeys,
-            { lookupOnly: lookupOnly },
-            enableCrossOsArchive
-        );
-
-        if (!cacheKey) {
-            if (failOnCacheMiss) {
-                throw new Error(
-                    `Failed to restore cache entry. Exiting as fail-on-cache-miss is set. Input key: ${primaryKey}`
-                );
-            }
-            core.info(
-                `Cache not found for input keys: ${[
-                    primaryKey,
-                    ...restoreKeys
-                ].join(", ")}`
-            );
-
+        const key = core.getInput(Inputs.Key, { required: true });
+        const bucket = core.getInput(Inputs.Bucket);
+        const workspace = process.env["GITHUB_WORKSPACE"] ?? process.cwd();
+        const exitCode = await exec("/bin/bash", [
+            "-c",
+            `gsutil -o 'GSUtil:parallel_thread_count=1' -o 'GSUtil:sliced_object_download_max_components=8' cp "gs://${bucket}/${key}" - | tar --skip-old-files -x -P -C "${workspace}"`
+        ]);
+        if (exitCode === 1) {
+            console.log("[warning]Failed to extract cache...");
             return;
         }
 
-        // Store the matched cache key in states
-        stateProvider.setState(State.CacheMatchedKey, cacheKey);
-
-        const isExactKeyMatch = utils.isExactKeyMatch(
-            core.getInput(Inputs.Key, { required: true }),
-            cacheKey
-        );
-
-        core.setOutput(Outputs.CacheHit, isExactKeyMatch.toString());
-        if (lookupOnly) {
-            core.info(`Cache found and can be restored from key: ${cacheKey}`);
-        } else {
-            core.info(`Cache restored from key: ${cacheKey}`);
-        }
-
-        return cacheKey;
+        // cache-id return set to 1
+        return "1";
     } catch (error: unknown) {
         core.setFailed((error as Error).message);
     }
 }
 
-async function run(
-    stateProvider: IStateProvider,
-    earlyExit: boolean | undefined
-): Promise<void> {
+async function run(earlyExit: boolean | undefined): Promise<void> {
     try {
-        await restoreImpl(stateProvider);
+        await restoreImpl();
     } catch (err) {
         console.error(err);
         if (earlyExit) {
@@ -112,11 +63,11 @@ async function run(
 export async function restoreOnlyRun(
     earlyExit?: boolean | undefined
 ): Promise<void> {
-    await run(new NullStateProvider(), earlyExit);
+    await run(earlyExit);
 }
 
 export async function restoreRun(
     earlyExit?: boolean | undefined
 ): Promise<void> {
-    await run(new StateProvider(), earlyExit);
+    await run(earlyExit);
 }

src/saveImpl.ts

@@ -1,8 +1,10 @@
-import * as cache from "@actions/cache";
+import * as cacheUtils from "@actions/cache/lib/internal/cacheUtils";
 import * as core from "@actions/core";
+import { exec } from "@actions/exec";
+import { writeFileSync } from "fs";
+import * as path from "path";
 
-import { Events, Inputs, State } from "./constants";
-import { IStateProvider } from "./stateProvider";
+import { Events, Inputs } from "./constants";
 import * as utils from "./utils/actionUtils";
 
 // Catch and log any unhandled exceptions.  These exceptions can leak out of the uploadChunk method in
@@ -10,8 +12,7 @@ import * as utils from "./utils/actionUtils";
 // throw an uncaught exception.  Instead of failing this action, just warn.
 process.on("uncaughtException", e => utils.logWarning(e.message));
 
-async function saveImpl(stateProvider: IStateProvider): Promise<number | void> {
-    let cacheId = -1;
+async function saveImpl(): Promise<number | void> {
     try {
         if (!utils.isCacheFeatureAvailable()) {
             return;
@@ -26,50 +27,32 @@ async function saveImpl(stateProvider: IStateProvider): Promise<number | void> {
             return;
         }
 
-        // If restore has stored a primary key in state, reuse that
-        // Else re-evaluate from inputs
-        const primaryKey =
-            stateProvider.getState(State.CachePrimaryKey) ||
-            core.getInput(Inputs.Key);
-
-        if (!primaryKey) {
-            utils.logWarning(`Key is not specified.`);
-            return;
-        }
-
-        // If matched restore key is same as primary key, then do not save cache
-        // NO-OP in case of SaveOnly action
-        const restoredKey = stateProvider.getCacheState();
-
-        if (utils.isExactKeyMatch(primaryKey, restoredKey)) {
-            core.info(
-                `Cache hit occurred on the primary key ${primaryKey}, not saving cache.`
-            );
-            return;
-        }
-
-        const cachePaths = utils.getInputAsArray(Inputs.Path, {
+        const key = core.getInput(Inputs.Key);
+        const bucket = core.getInput(Inputs.Bucket);
+        const paths = utils.getInputAsArray(Inputs.Path, {
             required: true
         });
 
-        const enableCrossOsArchive = utils.getInputAsBool(
-            Inputs.EnableCrossOsArchive
-        );
-
-        cacheId = await cache.saveCache(
-            cachePaths,
-            primaryKey,
-            { uploadChunkSize: utils.getInputAsInt(Inputs.UploadChunkSize) },
-            enableCrossOsArchive
-        );
-
-        if (cacheId != -1) {
-            core.info(`Cache saved with key: ${primaryKey}`);
+        // https://github.com/actions/toolkit/blob/c861dd8859fe5294289fcada363ce9bc71e9d260/packages/cache/src/internal/tar.ts#L75
+        const cachePaths = await cacheUtils.resolvePaths(paths);
+        const tmpFolder = await cacheUtils.createTempDirectory();
+        // Write source directories to manifest.txt to avoid command length limits
+        const manifestPath = path.join(tmpFolder, "manifest.txt");
+        writeFileSync(manifestPath, cachePaths.join("\n"));
+
+        const workspace = process.env["GITHUB_WORKSPACE"] ?? process.cwd();
+        const exitCode = await exec("/bin/bash", [
+            "-c",
+            `tar -cf - -P -C ${workspace} --files-from ${manifestPath} | gsutil -o 'GSUtil:parallel_composite_upload_threshold=250M' cp - "gs://${bucket}/${key}"`
+        ]);
+        if (exitCode !== 0) {
+            utils.logWarning("Failed to upload cache...");
         }
     } catch (error: unknown) {
         utils.logWarning((error as Error).message);
     }
-    return cacheId;
+    // cache-id return set to 1
+    return 1;
 }
 
 export default saveImpl;