mirror of https://github.com/actions/cache.git
Merge pull request #313 from actions/aiyan/use-cache-package
Switch cache action to use the cache node packagereleases/v2
Aiqiao Yan
5 years ago
committed by
GitHub
commit
9ab95382c8
@ -1,177 +0,0 @@
|
|||||||
import { getCacheVersion, retry } from "../src/cacheHttpClient";
|
|
||||||
import { CompressionMethod, Inputs } from "../src/constants";
|
|
||||||
import * as testUtils from "../src/utils/testUtils";
|
|
||||||
|
|
||||||
afterEach(() => {
|
|
||||||
testUtils.clearInputs();
|
|
||||||
});
|
|
||||||
|
|
||||||
test("getCacheVersion with path input and compression method undefined returns version", async () => {
|
|
||||||
testUtils.setInput(Inputs.Path, "node_modules");
|
|
||||||
|
|
||||||
const result = getCacheVersion();
|
|
||||||
|
|
||||||
expect(result).toEqual(
|
|
||||||
"b3e0c6cb5ecf32614eeb2997d905b9c297046d7cbf69062698f25b14b4cb0985"
|
|
||||||
);
|
|
||||||
});
|
|
||||||
|
|
||||||
test("getCacheVersion with zstd compression returns version", async () => {
|
|
||||||
testUtils.setInput(Inputs.Path, "node_modules");
|
|
||||||
const result = getCacheVersion(CompressionMethod.Zstd);
|
|
||||||
|
|
||||||
expect(result).toEqual(
|
|
||||||
"273877e14fd65d270b87a198edbfa2db5a43de567c9a548d2a2505b408befe24"
|
|
||||||
);
|
|
||||||
});
|
|
||||||
|
|
||||||
test("getCacheVersion with gzip compression does not change vesion", async () => {
|
|
||||||
testUtils.setInput(Inputs.Path, "node_modules");
|
|
||||||
const result = getCacheVersion(CompressionMethod.Gzip);
|
|
||||||
|
|
||||||
expect(result).toEqual(
|
|
||||||
"b3e0c6cb5ecf32614eeb2997d905b9c297046d7cbf69062698f25b14b4cb0985"
|
|
||||||
);
|
|
||||||
});
|
|
||||||
|
|
||||||
test("getCacheVersion with no input throws", async () => {
|
|
||||||
expect(() => getCacheVersion()).toThrow();
|
|
||||||
});
|
|
||||||
|
|
||||||
interface TestResponse {
|
|
||||||
statusCode: number;
|
|
||||||
result: string | null;
|
|
||||||
}
|
|
||||||
|
|
||||||
function handleResponse(
|
|
||||||
response: TestResponse | undefined
|
|
||||||
): Promise<TestResponse> {
|
|
||||||
if (!response) {
|
|
||||||
fail("Retry method called too many times");
|
|
||||||
}
|
|
||||||
|
|
||||||
if (response.statusCode === 999) {
|
|
||||||
throw Error("Test Error");
|
|
||||||
} else {
|
|
||||||
return Promise.resolve(response);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async function testRetryExpectingResult(
|
|
||||||
responses: Array<TestResponse>,
|
|
||||||
expectedResult: string | null
|
|
||||||
): Promise<void> {
|
|
||||||
responses = responses.reverse(); // Reverse responses since we pop from end
|
|
||||||
|
|
||||||
const actualResult = await retry(
|
|
||||||
"test",
|
|
||||||
() => handleResponse(responses.pop()),
|
|
||||||
(response: TestResponse) => response.statusCode
|
|
||||||
);
|
|
||||||
|
|
||||||
expect(actualResult.result).toEqual(expectedResult);
|
|
||||||
}
|
|
||||||
|
|
||||||
async function testRetryExpectingError(
|
|
||||||
responses: Array<TestResponse>
|
|
||||||
): Promise<void> {
|
|
||||||
responses = responses.reverse(); // Reverse responses since we pop from end
|
|
||||||
|
|
||||||
expect(
|
|
||||||
retry(
|
|
||||||
"test",
|
|
||||||
() => handleResponse(responses.pop()),
|
|
||||||
(response: TestResponse) => response.statusCode
|
|
||||||
)
|
|
||||||
).rejects.toBeInstanceOf(Error);
|
|
||||||
}
|
|
||||||
|
|
||||||
test("retry works on successful response", async () => {
|
|
||||||
await testRetryExpectingResult(
|
|
||||||
[
|
|
||||||
{
|
|
||||||
statusCode: 200,
|
|
||||||
result: "Ok"
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"Ok"
|
|
||||||
);
|
|
||||||
});
|
|
||||||
|
|
||||||
test("retry works after retryable status code", async () => {
|
|
||||||
await testRetryExpectingResult(
|
|
||||||
[
|
|
||||||
{
|
|
||||||
statusCode: 503,
|
|
||||||
result: null
|
|
||||||
},
|
|
||||||
{
|
|
||||||
statusCode: 200,
|
|
||||||
result: "Ok"
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"Ok"
|
|
||||||
);
|
|
||||||
});
|
|
||||||
|
|
||||||
test("retry fails after exhausting retries", async () => {
|
|
||||||
await testRetryExpectingError([
|
|
||||||
{
|
|
||||||
statusCode: 503,
|
|
||||||
result: null
|
|
||||||
},
|
|
||||||
{
|
|
||||||
statusCode: 503,
|
|
||||||
result: null
|
|
||||||
},
|
|
||||||
{
|
|
||||||
statusCode: 200,
|
|
||||||
result: "Ok"
|
|
||||||
}
|
|
||||||
]);
|
|
||||||
});
|
|
||||||
|
|
||||||
test("retry fails after non-retryable status code", async () => {
|
|
||||||
await testRetryExpectingError([
|
|
||||||
{
|
|
||||||
statusCode: 500,
|
|
||||||
result: null
|
|
||||||
},
|
|
||||||
{
|
|
||||||
statusCode: 200,
|
|
||||||
result: "Ok"
|
|
||||||
}
|
|
||||||
]);
|
|
||||||
});
|
|
||||||
|
|
||||||
test("retry works after error", async () => {
|
|
||||||
await testRetryExpectingResult(
|
|
||||||
[
|
|
||||||
{
|
|
||||||
statusCode: 999,
|
|
||||||
result: null
|
|
||||||
},
|
|
||||||
{
|
|
||||||
statusCode: 200,
|
|
||||||
result: "Ok"
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"Ok"
|
|
||||||
);
|
|
||||||
});
|
|
||||||
|
|
||||||
test("retry returns after client error", async () => {
|
|
||||||
await testRetryExpectingResult(
|
|
||||||
[
|
|
||||||
{
|
|
||||||
statusCode: 400,
|
|
||||||
result: null
|
|
||||||
},
|
|
||||||
{
|
|
||||||
statusCode: 200,
|
|
||||||
result: "Ok"
|
|
||||||
}
|
|
||||||
],
|
|
||||||
null
|
|
||||||
);
|
|
||||||
});
|
|
||||||
@ -1,204 +0,0 @@
|
|||||||
import * as exec from "@actions/exec";
|
|
||||||
import * as io from "@actions/io";
|
|
||||||
import * as path from "path";
|
|
||||||
|
|
||||||
import { CacheFilename, CompressionMethod } from "../src/constants";
|
|
||||||
import * as tar from "../src/tar";
|
|
||||||
import * as utils from "../src/utils/actionUtils";
|
|
||||||
|
|
||||||
import fs = require("fs");
|
|
||||||
|
|
||||||
jest.mock("@actions/exec");
|
|
||||||
jest.mock("@actions/io");
|
|
||||||
|
|
||||||
const IS_WINDOWS = process.platform === "win32";
|
|
||||||
|
|
||||||
function getTempDir(): string {
|
|
||||||
return path.join(__dirname, "_temp", "tar");
|
|
||||||
}
|
|
||||||
|
|
||||||
beforeAll(async () => {
|
|
||||||
jest.spyOn(io, "which").mockImplementation(tool => {
|
|
||||||
return Promise.resolve(tool);
|
|
||||||
});
|
|
||||||
|
|
||||||
process.env["GITHUB_WORKSPACE"] = process.cwd();
|
|
||||||
await jest.requireActual("@actions/io").rmRF(getTempDir());
|
|
||||||
});
|
|
||||||
|
|
||||||
afterAll(async () => {
|
|
||||||
delete process.env["GITHUB_WORKSPACE"];
|
|
||||||
await jest.requireActual("@actions/io").rmRF(getTempDir());
|
|
||||||
});
|
|
||||||
|
|
||||||
test("zstd extract tar", async () => {
|
|
||||||
const mkdirMock = jest.spyOn(io, "mkdirP");
|
|
||||||
const execMock = jest.spyOn(exec, "exec");
|
|
||||||
|
|
||||||
const archivePath = IS_WINDOWS
|
|
||||||
? `${process.env["windir"]}\\fakepath\\cache.tar`
|
|
||||||
: "cache.tar";
|
|
||||||
const workspace = process.env["GITHUB_WORKSPACE"];
|
|
||||||
|
|
||||||
await tar.extractTar(archivePath, CompressionMethod.Zstd);
|
|
||||||
|
|
||||||
expect(mkdirMock).toHaveBeenCalledWith(workspace);
|
|
||||||
const tarPath = IS_WINDOWS
|
|
||||||
? `${process.env["windir"]}\\System32\\tar.exe`
|
|
||||||
: "tar";
|
|
||||||
expect(execMock).toHaveBeenCalledTimes(1);
|
|
||||||
expect(execMock).toHaveBeenCalledWith(
|
|
||||||
`"${tarPath}"`,
|
|
||||||
[
|
|
||||||
"--use-compress-program",
|
|
||||||
"zstd -d --long=30",
|
|
||||||
"-xf",
|
|
||||||
IS_WINDOWS ? archivePath.replace(/\\/g, "/") : archivePath,
|
|
||||||
"-P",
|
|
||||||
"-C",
|
|
||||||
IS_WINDOWS ? workspace?.replace(/\\/g, "/") : workspace
|
|
||||||
],
|
|
||||||
{ cwd: undefined }
|
|
||||||
);
|
|
||||||
});
|
|
||||||
|
|
||||||
test("gzip extract tar", async () => {
|
|
||||||
const mkdirMock = jest.spyOn(io, "mkdirP");
|
|
||||||
const execMock = jest.spyOn(exec, "exec");
|
|
||||||
const archivePath = IS_WINDOWS
|
|
||||||
? `${process.env["windir"]}\\fakepath\\cache.tar`
|
|
||||||
: "cache.tar";
|
|
||||||
const workspace = process.env["GITHUB_WORKSPACE"];
|
|
||||||
|
|
||||||
await tar.extractTar(archivePath, CompressionMethod.Gzip);
|
|
||||||
|
|
||||||
expect(mkdirMock).toHaveBeenCalledWith(workspace);
|
|
||||||
const tarPath = IS_WINDOWS
|
|
||||||
? `${process.env["windir"]}\\System32\\tar.exe`
|
|
||||||
: "tar";
|
|
||||||
expect(execMock).toHaveBeenCalledTimes(1);
|
|
||||||
expect(execMock).toHaveBeenCalledWith(
|
|
||||||
`"${tarPath}"`,
|
|
||||||
[
|
|
||||||
"-z",
|
|
||||||
"-xf",
|
|
||||||
IS_WINDOWS ? archivePath.replace(/\\/g, "/") : archivePath,
|
|
||||||
"-P",
|
|
||||||
"-C",
|
|
||||||
IS_WINDOWS ? workspace?.replace(/\\/g, "/") : workspace
|
|
||||||
],
|
|
||||||
{ cwd: undefined }
|
|
||||||
);
|
|
||||||
});
|
|
||||||
|
|
||||||
test("gzip extract GNU tar on windows", async () => {
|
|
||||||
if (IS_WINDOWS) {
|
|
||||||
jest.spyOn(fs, "existsSync").mockReturnValueOnce(false);
|
|
||||||
|
|
||||||
const isGnuMock = jest
|
|
||||||
.spyOn(utils, "useGnuTar")
|
|
||||||
.mockReturnValue(Promise.resolve(true));
|
|
||||||
const execMock = jest.spyOn(exec, "exec");
|
|
||||||
const archivePath = `${process.env["windir"]}\\fakepath\\cache.tar`;
|
|
||||||
const workspace = process.env["GITHUB_WORKSPACE"];
|
|
||||||
|
|
||||||
await tar.extractTar(archivePath, CompressionMethod.Gzip);
|
|
||||||
|
|
||||||
expect(isGnuMock).toHaveBeenCalledTimes(1);
|
|
||||||
expect(execMock).toHaveBeenCalledTimes(1);
|
|
||||||
expect(execMock).toHaveBeenCalledWith(
|
|
||||||
`"tar"`,
|
|
||||||
[
|
|
||||||
"-z",
|
|
||||||
"-xf",
|
|
||||||
archivePath.replace(/\\/g, "/"),
|
|
||||||
"-P",
|
|
||||||
"-C",
|
|
||||||
workspace?.replace(/\\/g, "/"),
|
|
||||||
"--force-local"
|
|
||||||
],
|
|
||||||
{ cwd: undefined }
|
|
||||||
);
|
|
||||||
}
|
|
||||||
});
|
|
||||||
|
|
||||||
test("zstd create tar", async () => {
|
|
||||||
const execMock = jest.spyOn(exec, "exec");
|
|
||||||
|
|
||||||
const archiveFolder = getTempDir();
|
|
||||||
const workspace = process.env["GITHUB_WORKSPACE"];
|
|
||||||
const sourceDirectories = ["~/.npm/cache", `${workspace}/dist`];
|
|
||||||
|
|
||||||
await fs.promises.mkdir(archiveFolder, { recursive: true });
|
|
||||||
|
|
||||||
await tar.createTar(
|
|
||||||
archiveFolder,
|
|
||||||
sourceDirectories,
|
|
||||||
CompressionMethod.Zstd
|
|
||||||
);
|
|
||||||
|
|
||||||
const tarPath = IS_WINDOWS
|
|
||||||
? `${process.env["windir"]}\\System32\\tar.exe`
|
|
||||||
: "tar";
|
|
||||||
|
|
||||||
expect(execMock).toHaveBeenCalledTimes(1);
|
|
||||||
expect(execMock).toHaveBeenCalledWith(
|
|
||||||
`"${tarPath}"`,
|
|
||||||
[
|
|
||||||
"--use-compress-program",
|
|
||||||
"zstd -T0 --long=30",
|
|
||||||
"-cf",
|
|
||||||
IS_WINDOWS
|
|
||||||
? CacheFilename.Zstd.replace(/\\/g, "/")
|
|
||||||
: CacheFilename.Zstd,
|
|
||||||
"-P",
|
|
||||||
"-C",
|
|
||||||
IS_WINDOWS ? workspace?.replace(/\\/g, "/") : workspace,
|
|
||||||
"--files-from",
|
|
||||||
"manifest.txt"
|
|
||||||
],
|
|
||||||
{
|
|
||||||
cwd: archiveFolder
|
|
||||||
}
|
|
||||||
);
|
|
||||||
});
|
|
||||||
|
|
||||||
test("gzip create tar", async () => {
|
|
||||||
const execMock = jest.spyOn(exec, "exec");
|
|
||||||
|
|
||||||
const archiveFolder = getTempDir();
|
|
||||||
const workspace = process.env["GITHUB_WORKSPACE"];
|
|
||||||
const sourceDirectories = ["~/.npm/cache", `${workspace}/dist`];
|
|
||||||
|
|
||||||
await fs.promises.mkdir(archiveFolder, { recursive: true });
|
|
||||||
|
|
||||||
await tar.createTar(
|
|
||||||
archiveFolder,
|
|
||||||
sourceDirectories,
|
|
||||||
CompressionMethod.Gzip
|
|
||||||
);
|
|
||||||
|
|
||||||
const tarPath = IS_WINDOWS
|
|
||||||
? `${process.env["windir"]}\\System32\\tar.exe`
|
|
||||||
: "tar";
|
|
||||||
|
|
||||||
expect(execMock).toHaveBeenCalledTimes(1);
|
|
||||||
expect(execMock).toHaveBeenCalledWith(
|
|
||||||
`"${tarPath}"`,
|
|
||||||
[
|
|
||||||
"-z",
|
|
||||||
"-cf",
|
|
||||||
IS_WINDOWS
|
|
||||||
? CacheFilename.Gzip.replace(/\\/g, "/")
|
|
||||||
: CacheFilename.Gzip,
|
|
||||||
"-P",
|
|
||||||
"-C",
|
|
||||||
IS_WINDOWS ? workspace?.replace(/\\/g, "/") : workspace,
|
|
||||||
"--files-from",
|
|
||||||
"manifest.txt"
|
|
||||||
],
|
|
||||||
{
|
|
||||||
cwd: archiveFolder
|
|
||||||
}
|
|
||||||
);
|
|
||||||
});
|
|
||||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@ -1,424 +0,0 @@
|
|||||||
import * as core from "@actions/core";
|
|
||||||
import { HttpClient, HttpCodes } from "@actions/http-client";
|
|
||||||
import { BearerCredentialHandler } from "@actions/http-client/auth";
|
|
||||||
import {
|
|
||||||
IHttpClientResponse,
|
|
||||||
IRequestOptions,
|
|
||||||
ITypedResponse
|
|
||||||
} from "@actions/http-client/interfaces";
|
|
||||||
import * as crypto from "crypto";
|
|
||||||
import * as fs from "fs";
|
|
||||||
import * as stream from "stream";
|
|
||||||
import * as util from "util";
|
|
||||||
|
|
||||||
import { CompressionMethod, Inputs, SocketTimeout } from "./constants";
|
|
||||||
import {
|
|
||||||
ArtifactCacheEntry,
|
|
||||||
CacheOptions,
|
|
||||||
CommitCacheRequest,
|
|
||||||
ReserveCacheRequest,
|
|
||||||
ReserveCacheResponse
|
|
||||||
} from "./contracts";
|
|
||||||
import * as utils from "./utils/actionUtils";
|
|
||||||
|
|
||||||
const versionSalt = "1.0";
|
|
||||||
|
|
||||||
function isSuccessStatusCode(statusCode?: number): boolean {
|
|
||||||
if (!statusCode) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
return statusCode >= 200 && statusCode < 300;
|
|
||||||
}
|
|
||||||
|
|
||||||
function isServerErrorStatusCode(statusCode?: number): boolean {
|
|
||||||
if (!statusCode) {
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
return statusCode >= 500;
|
|
||||||
}
|
|
||||||
|
|
||||||
function isRetryableStatusCode(statusCode?: number): boolean {
|
|
||||||
if (!statusCode) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
const retryableStatusCodes = [
|
|
||||||
HttpCodes.BadGateway,
|
|
||||||
HttpCodes.ServiceUnavailable,
|
|
||||||
HttpCodes.GatewayTimeout
|
|
||||||
];
|
|
||||||
return retryableStatusCodes.includes(statusCode);
|
|
||||||
}
|
|
||||||
|
|
||||||
function getCacheApiUrl(resource: string): string {
|
|
||||||
// Ideally we just use ACTIONS_CACHE_URL
|
|
||||||
const baseUrl: string = (
|
|
||||||
process.env["ACTIONS_CACHE_URL"] ||
|
|
||||||
process.env["ACTIONS_RUNTIME_URL"] ||
|
|
||||||
""
|
|
||||||
).replace("pipelines", "artifactcache");
|
|
||||||
if (!baseUrl) {
|
|
||||||
throw new Error(
|
|
||||||
"Cache Service Url not found, unable to restore cache."
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
const url = `${baseUrl}_apis/artifactcache/${resource}`;
|
|
||||||
core.debug(`Resource Url: ${url}`);
|
|
||||||
return url;
|
|
||||||
}
|
|
||||||
|
|
||||||
function createAcceptHeader(type: string, apiVersion: string): string {
|
|
||||||
return `${type};api-version=${apiVersion}`;
|
|
||||||
}
|
|
||||||
|
|
||||||
function getRequestOptions(): IRequestOptions {
|
|
||||||
const requestOptions: IRequestOptions = {
|
|
||||||
headers: {
|
|
||||||
Accept: createAcceptHeader("application/json", "6.0-preview.1")
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
return requestOptions;
|
|
||||||
}
|
|
||||||
|
|
||||||
function createHttpClient(): HttpClient {
|
|
||||||
const token = process.env["ACTIONS_RUNTIME_TOKEN"] || "";
|
|
||||||
const bearerCredentialHandler = new BearerCredentialHandler(token);
|
|
||||||
|
|
||||||
return new HttpClient(
|
|
||||||
"actions/cache",
|
|
||||||
[bearerCredentialHandler],
|
|
||||||
getRequestOptions()
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
export function getCacheVersion(compressionMethod?: CompressionMethod): string {
|
|
||||||
const components = [core.getInput(Inputs.Path, { required: true })].concat(
|
|
||||||
compressionMethod == CompressionMethod.Zstd ? [compressionMethod] : []
|
|
||||||
);
|
|
||||||
|
|
||||||
// Add salt to cache version to support breaking changes in cache entry
|
|
||||||
components.push(versionSalt);
|
|
||||||
|
|
||||||
return crypto
|
|
||||||
.createHash("sha256")
|
|
||||||
.update(components.join("|"))
|
|
||||||
.digest("hex");
|
|
||||||
}
|
|
||||||
|
|
||||||
export async function retry<T>(
|
|
||||||
name: string,
|
|
||||||
method: () => Promise<T>,
|
|
||||||
getStatusCode: (T) => number | undefined,
|
|
||||||
maxAttempts = 2
|
|
||||||
): Promise<T> {
|
|
||||||
let response: T | undefined = undefined;
|
|
||||||
let statusCode: number | undefined = undefined;
|
|
||||||
let isRetryable = false;
|
|
||||||
let errorMessage = "";
|
|
||||||
let attempt = 1;
|
|
||||||
|
|
||||||
while (attempt <= maxAttempts) {
|
|
||||||
try {
|
|
||||||
response = await method();
|
|
||||||
statusCode = getStatusCode(response);
|
|
||||||
|
|
||||||
if (!isServerErrorStatusCode(statusCode)) {
|
|
||||||
return response;
|
|
||||||
}
|
|
||||||
|
|
||||||
isRetryable = isRetryableStatusCode(statusCode);
|
|
||||||
errorMessage = `Cache service responded with ${statusCode}`;
|
|
||||||
} catch (error) {
|
|
||||||
isRetryable = true;
|
|
||||||
errorMessage = error.message;
|
|
||||||
}
|
|
||||||
|
|
||||||
core.debug(
|
|
||||||
`${name} - Attempt ${attempt} of ${maxAttempts} failed with error: ${errorMessage}`
|
|
||||||
);
|
|
||||||
|
|
||||||
if (!isRetryable) {
|
|
||||||
core.debug(`${name} - Error is not retryable`);
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
|
|
||||||
attempt++;
|
|
||||||
}
|
|
||||||
|
|
||||||
throw Error(`${name} failed: ${errorMessage}`);
|
|
||||||
}
|
|
||||||
|
|
||||||
export async function retryTypedResponse<T>(
|
|
||||||
name: string,
|
|
||||||
method: () => Promise<ITypedResponse<T>>,
|
|
||||||
maxAttempts = 2
|
|
||||||
): Promise<ITypedResponse<T>> {
|
|
||||||
return await retry(
|
|
||||||
name,
|
|
||||||
method,
|
|
||||||
(response: ITypedResponse<T>) => response.statusCode,
|
|
||||||
maxAttempts
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
export async function retryHttpClientResponse<T>(
|
|
||||||
name: string,
|
|
||||||
method: () => Promise<IHttpClientResponse>,
|
|
||||||
maxAttempts = 2
|
|
||||||
): Promise<IHttpClientResponse> {
|
|
||||||
return await retry(
|
|
||||||
name,
|
|
||||||
method,
|
|
||||||
(response: IHttpClientResponse) => response.message.statusCode,
|
|
||||||
maxAttempts
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
export async function getCacheEntry(
|
|
||||||
keys: string[],
|
|
||||||
options?: CacheOptions
|
|
||||||
): Promise<ArtifactCacheEntry | null> {
|
|
||||||
const httpClient = createHttpClient();
|
|
||||||
const version = getCacheVersion(options?.compressionMethod);
|
|
||||||
const resource = `cache?keys=${encodeURIComponent(
|
|
||||||
keys.join(",")
|
|
||||||
)}&version=${version}`;
|
|
||||||
|
|
||||||
const response = await retryTypedResponse("getCacheEntry", () =>
|
|
||||||
httpClient.getJson<ArtifactCacheEntry>(getCacheApiUrl(resource))
|
|
||||||
);
|
|
||||||
if (response.statusCode === 204) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
if (!isSuccessStatusCode(response.statusCode)) {
|
|
||||||
throw new Error(`Cache service responded with ${response.statusCode}`);
|
|
||||||
}
|
|
||||||
|
|
||||||
const cacheResult = response.result;
|
|
||||||
const cacheDownloadUrl = cacheResult?.archiveLocation;
|
|
||||||
if (!cacheDownloadUrl) {
|
|
||||||
throw new Error("Cache not found.");
|
|
||||||
}
|
|
||||||
core.setSecret(cacheDownloadUrl);
|
|
||||||
core.debug(`Cache Result:`);
|
|
||||||
core.debug(JSON.stringify(cacheResult));
|
|
||||||
|
|
||||||
return cacheResult;
|
|
||||||
}
|
|
||||||
|
|
||||||
async function pipeResponseToStream(
|
|
||||||
response: IHttpClientResponse,
|
|
||||||
output: NodeJS.WritableStream
|
|
||||||
): Promise<void> {
|
|
||||||
const pipeline = util.promisify(stream.pipeline);
|
|
||||||
await pipeline(response.message, output);
|
|
||||||
}
|
|
||||||
|
|
||||||
export async function downloadCache(
|
|
||||||
archiveLocation: string,
|
|
||||||
archivePath: string
|
|
||||||
): Promise<void> {
|
|
||||||
const stream = fs.createWriteStream(archivePath);
|
|
||||||
const httpClient = new HttpClient("actions/cache");
|
|
||||||
const downloadResponse = await retryHttpClientResponse(
|
|
||||||
"downloadCache",
|
|
||||||
() => httpClient.get(archiveLocation)
|
|
||||||
);
|
|
||||||
|
|
||||||
// Abort download if no traffic received over the socket.
|
|
||||||
downloadResponse.message.socket.setTimeout(SocketTimeout, () => {
|
|
||||||
downloadResponse.message.destroy();
|
|
||||||
core.debug(
|
|
||||||
`Aborting download, socket timed out after ${SocketTimeout} ms`
|
|
||||||
);
|
|
||||||
});
|
|
||||||
|
|
||||||
await pipeResponseToStream(downloadResponse, stream);
|
|
||||||
|
|
||||||
// Validate download size.
|
|
||||||
const contentLengthHeader =
|
|
||||||
downloadResponse.message.headers["content-length"];
|
|
||||||
|
|
||||||
if (contentLengthHeader) {
|
|
||||||
const expectedLength = parseInt(contentLengthHeader);
|
|
||||||
const actualLength = utils.getArchiveFileSize(archivePath);
|
|
||||||
|
|
||||||
if (actualLength != expectedLength) {
|
|
||||||
throw new Error(
|
|
||||||
`Incomplete download. Expected file size: ${expectedLength}, actual file size: ${actualLength}`
|
|
||||||
);
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
core.debug("Unable to validate download, no Content-Length header");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Reserve Cache
|
|
||||||
export async function reserveCache(
|
|
||||||
key: string,
|
|
||||||
options?: CacheOptions
|
|
||||||
): Promise<number> {
|
|
||||||
const httpClient = createHttpClient();
|
|
||||||
const version = getCacheVersion(options?.compressionMethod);
|
|
||||||
|
|
||||||
const reserveCacheRequest: ReserveCacheRequest = {
|
|
||||||
key,
|
|
||||||
version
|
|
||||||
};
|
|
||||||
const response = await retryTypedResponse("reserveCache", () =>
|
|
||||||
httpClient.postJson<ReserveCacheResponse>(
|
|
||||||
getCacheApiUrl("caches"),
|
|
||||||
reserveCacheRequest
|
|
||||||
)
|
|
||||||
);
|
|
||||||
return response?.result?.cacheId ?? -1;
|
|
||||||
}
|
|
||||||
|
|
||||||
function getContentRange(start: number, end: number): string {
|
|
||||||
// Format: `bytes start-end/filesize
|
|
||||||
// start and end are inclusive
|
|
||||||
// filesize can be *
|
|
||||||
// For a 200 byte chunk starting at byte 0:
|
|
||||||
// Content-Range: bytes 0-199/*
|
|
||||||
return `bytes ${start}-${end}/*`;
|
|
||||||
}
|
|
||||||
|
|
||||||
async function uploadChunk(
|
|
||||||
httpClient: HttpClient,
|
|
||||||
resourceUrl: string,
|
|
||||||
openStream: () => NodeJS.ReadableStream,
|
|
||||||
start: number,
|
|
||||||
end: number
|
|
||||||
): Promise<void> {
|
|
||||||
core.debug(
|
|
||||||
`Uploading chunk of size ${end -
|
|
||||||
start +
|
|
||||||
1} bytes at offset ${start} with content range: ${getContentRange(
|
|
||||||
start,
|
|
||||||
end
|
|
||||||
)}`
|
|
||||||
);
|
|
||||||
const additionalHeaders = {
|
|
||||||
"Content-Type": "application/octet-stream",
|
|
||||||
"Content-Range": getContentRange(start, end)
|
|
||||||
};
|
|
||||||
|
|
||||||
await retryHttpClientResponse(
|
|
||||||
`uploadChunk (start: ${start}, end: ${end})`,
|
|
||||||
() =>
|
|
||||||
httpClient.sendStream(
|
|
||||||
"PATCH",
|
|
||||||
resourceUrl,
|
|
||||||
openStream(),
|
|
||||||
additionalHeaders
|
|
||||||
)
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
function parseEnvNumber(key: string): number | undefined {
|
|
||||||
const value = Number(process.env[key]);
|
|
||||||
if (Number.isNaN(value) || value < 0) {
|
|
||||||
return undefined;
|
|
||||||
}
|
|
||||||
return value;
|
|
||||||
}
|
|
||||||
|
|
||||||
async function uploadFile(
|
|
||||||
httpClient: HttpClient,
|
|
||||||
cacheId: number,
|
|
||||||
archivePath: string
|
|
||||||
): Promise<void> {
|
|
||||||
// Upload Chunks
|
|
||||||
const fileSize = fs.statSync(archivePath).size;
|
|
||||||
const resourceUrl = getCacheApiUrl(`caches/${cacheId.toString()}`);
|
|
||||||
const fd = fs.openSync(archivePath, "r");
|
|
||||||
|
|
||||||
const concurrency = parseEnvNumber("CACHE_UPLOAD_CONCURRENCY") ?? 4; // # of HTTP requests in parallel
|
|
||||||
const MAX_CHUNK_SIZE =
|
|
||||||
parseEnvNumber("CACHE_UPLOAD_CHUNK_SIZE") ?? 32 * 1024 * 1024; // 32 MB Chunks
|
|
||||||
core.debug(`Concurrency: ${concurrency} and Chunk Size: ${MAX_CHUNK_SIZE}`);
|
|
||||||
|
|
||||||
const parallelUploads = [...new Array(concurrency).keys()];
|
|
||||||
core.debug("Awaiting all uploads");
|
|
||||||
let offset = 0;
|
|
||||||
|
|
||||||
try {
|
|
||||||
await Promise.all(
|
|
||||||
parallelUploads.map(async () => {
|
|
||||||
while (offset < fileSize) {
|
|
||||||
const chunkSize = Math.min(
|
|
||||||
fileSize - offset,
|
|
||||||
MAX_CHUNK_SIZE
|
|
||||||
);
|
|
||||||
const start = offset;
|
|
||||||
const end = offset + chunkSize - 1;
|
|
||||||
offset += MAX_CHUNK_SIZE;
|
|
||||||
|
|
||||||
await uploadChunk(
|
|
||||||
httpClient,
|
|
||||||
resourceUrl,
|
|
||||||
() =>
|
|
||||||
fs
|
|
||||||
.createReadStream(archivePath, {
|
|
||||||
fd,
|
|
||||||
start,
|
|
||||||
end,
|
|
||||||
autoClose: false
|
|
||||||
})
|
|
||||||
.on("error", error => {
|
|
||||||
throw new Error(
|
|
||||||
`Cache upload failed because file read failed with ${error.Message}`
|
|
||||||
);
|
|
||||||
}),
|
|
||||||
start,
|
|
||||||
end
|
|
||||||
);
|
|
||||||
}
|
|
||||||
})
|
|
||||||
);
|
|
||||||
} finally {
|
|
||||||
fs.closeSync(fd);
|
|
||||||
}
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
async function commitCache(
|
|
||||||
httpClient: HttpClient,
|
|
||||||
cacheId: number,
|
|
||||||
filesize: number
|
|
||||||
): Promise<ITypedResponse<null>> {
|
|
||||||
const commitCacheRequest: CommitCacheRequest = { size: filesize };
|
|
||||||
return await retryTypedResponse("commitCache", () =>
|
|
||||||
httpClient.postJson<null>(
|
|
||||||
getCacheApiUrl(`caches/${cacheId.toString()}`),
|
|
||||||
commitCacheRequest
|
|
||||||
)
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
export async function saveCache(
|
|
||||||
cacheId: number,
|
|
||||||
archivePath: string
|
|
||||||
): Promise<void> {
|
|
||||||
const httpClient = createHttpClient();
|
|
||||||
|
|
||||||
core.debug("Upload cache");
|
|
||||||
await uploadFile(httpClient, cacheId, archivePath);
|
|
||||||
|
|
||||||
// Commit Cache
|
|
||||||
core.debug("Commiting cache");
|
|
||||||
const cacheSize = utils.getArchiveFileSize(archivePath);
|
|
||||||
const commitCacheResponse = await commitCache(
|
|
||||||
httpClient,
|
|
||||||
cacheId,
|
|
||||||
cacheSize
|
|
||||||
);
|
|
||||||
if (!isSuccessStatusCode(commitCacheResponse.statusCode)) {
|
|
||||||
throw new Error(
|
|
||||||
`Cache service responded with ${commitCacheResponse.statusCode} during commit cache.`
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
core.info("Cache saved successfully");
|
|
||||||
}
|
|
||||||
@ -1,25 +0,0 @@
|
|||||||
import { CompressionMethod } from "./constants";
|
|
||||||
|
|
||||||
export interface ArtifactCacheEntry {
|
|
||||||
cacheKey?: string;
|
|
||||||
scope?: string;
|
|
||||||
creationTime?: string;
|
|
||||||
archiveLocation?: string;
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface CommitCacheRequest {
|
|
||||||
size: number;
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface ReserveCacheRequest {
|
|
||||||
key: string;
|
|
||||||
version?: string;
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface ReserveCacheResponse {
|
|
||||||
cacheId: number;
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface CacheOptions {
|
|
||||||
compressionMethod?: CompressionMethod;
|
|
||||||
}
|
|
||||||
@ -1,87 +0,0 @@
|
|||||||
import { exec } from "@actions/exec";
|
|
||||||
import * as io from "@actions/io";
|
|
||||||
import { existsSync, writeFileSync } from "fs";
|
|
||||||
import * as path from "path";
|
|
||||||
|
|
||||||
import { CompressionMethod } from "./constants";
|
|
||||||
import * as utils from "./utils/actionUtils";
|
|
||||||
|
|
||||||
async function getTarPath(args: string[]): Promise<string> {
|
|
||||||
// Explicitly use BSD Tar on Windows
|
|
||||||
const IS_WINDOWS = process.platform === "win32";
|
|
||||||
if (IS_WINDOWS) {
|
|
||||||
const systemTar = `${process.env["windir"]}\\System32\\tar.exe`;
|
|
||||||
if (existsSync(systemTar)) {
|
|
||||||
return systemTar;
|
|
||||||
} else if (await utils.useGnuTar()) {
|
|
||||||
args.push("--force-local");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return await io.which("tar", true);
|
|
||||||
}
|
|
||||||
|
|
||||||
async function execTar(args: string[], cwd?: string): Promise<void> {
|
|
||||||
try {
|
|
||||||
await exec(`"${await getTarPath(args)}"`, args, { cwd: cwd });
|
|
||||||
} catch (error) {
|
|
||||||
throw new Error(`Tar failed with error: ${error?.message}`);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
function getWorkingDirectory(): string {
|
|
||||||
return process.env["GITHUB_WORKSPACE"] ?? process.cwd();
|
|
||||||
}
|
|
||||||
|
|
||||||
export async function extractTar(
|
|
||||||
archivePath: string,
|
|
||||||
compressionMethod: CompressionMethod
|
|
||||||
): Promise<void> {
|
|
||||||
// Create directory to extract tar into
|
|
||||||
const workingDirectory = getWorkingDirectory();
|
|
||||||
await io.mkdirP(workingDirectory);
|
|
||||||
// --d: Decompress.
|
|
||||||
// --long=#: Enables long distance matching with # bits. Maximum is 30 (1GB) on 32-bit OS and 31 (2GB) on 64-bit.
|
|
||||||
// Using 30 here because we also support 32-bit self-hosted runners.
|
|
||||||
const args = [
|
|
||||||
...(compressionMethod == CompressionMethod.Zstd
|
|
||||||
? ["--use-compress-program", "zstd -d --long=30"]
|
|
||||||
: ["-z"]),
|
|
||||||
"-xf",
|
|
||||||
archivePath.replace(new RegExp("\\" + path.sep, "g"), "/"),
|
|
||||||
"-P",
|
|
||||||
"-C",
|
|
||||||
workingDirectory.replace(new RegExp("\\" + path.sep, "g"), "/")
|
|
||||||
];
|
|
||||||
await execTar(args);
|
|
||||||
}
|
|
||||||
|
|
||||||
export async function createTar(
|
|
||||||
archiveFolder: string,
|
|
||||||
sourceDirectories: string[],
|
|
||||||
compressionMethod: CompressionMethod
|
|
||||||
): Promise<void> {
|
|
||||||
// Write source directories to manifest.txt to avoid command length limits
|
|
||||||
const manifestFilename = "manifest.txt";
|
|
||||||
const cacheFileName = utils.getCacheFileName(compressionMethod);
|
|
||||||
writeFileSync(
|
|
||||||
path.join(archiveFolder, manifestFilename),
|
|
||||||
sourceDirectories.join("\n")
|
|
||||||
);
|
|
||||||
// -T#: Compress using # working thread. If # is 0, attempt to detect and use the number of physical CPU cores.
|
|
||||||
// --long=#: Enables long distance matching with # bits. Maximum is 30 (1GB) on 32-bit OS and 31 (2GB) on 64-bit.
|
|
||||||
// Using 30 here because we also support 32-bit self-hosted runners.
|
|
||||||
const workingDirectory = getWorkingDirectory();
|
|
||||||
const args = [
|
|
||||||
...(compressionMethod == CompressionMethod.Zstd
|
|
||||||
? ["--use-compress-program", "zstd -T0 --long=30"]
|
|
||||||
: ["-z"]),
|
|
||||||
"-cf",
|
|
||||||
cacheFileName.replace(new RegExp("\\" + path.sep, "g"), "/"),
|
|
||||||
"-P",
|
|
||||||
"-C",
|
|
||||||
workingDirectory.replace(new RegExp("\\" + path.sep, "g"), "/"),
|
|
||||||
"--files-from",
|
|
||||||
manifestFilename
|
|
||||||
];
|
|
||||||
await execTar(args, archiveFolder);
|
|
||||||
}
|
|
||||||
Loading…
Reference in New Issue