mirror of https://github.com/actions/cache.git
Merge pull request #313 from actions/aiyan/use-cache-package
Switch cache action to use the cache node packagereleases/v2
Aiqiao Yan
5 years ago
committed by
GitHub
commit
9ab95382c8
@ -1,177 +0,0 @@
|
||||
import { getCacheVersion, retry } from "../src/cacheHttpClient";
|
||||
import { CompressionMethod, Inputs } from "../src/constants";
|
||||
import * as testUtils from "../src/utils/testUtils";
|
||||
|
||||
afterEach(() => {
|
||||
testUtils.clearInputs();
|
||||
});
|
||||
|
||||
test("getCacheVersion with path input and compression method undefined returns version", async () => {
|
||||
testUtils.setInput(Inputs.Path, "node_modules");
|
||||
|
||||
const result = getCacheVersion();
|
||||
|
||||
expect(result).toEqual(
|
||||
"b3e0c6cb5ecf32614eeb2997d905b9c297046d7cbf69062698f25b14b4cb0985"
|
||||
);
|
||||
});
|
||||
|
||||
test("getCacheVersion with zstd compression returns version", async () => {
|
||||
testUtils.setInput(Inputs.Path, "node_modules");
|
||||
const result = getCacheVersion(CompressionMethod.Zstd);
|
||||
|
||||
expect(result).toEqual(
|
||||
"273877e14fd65d270b87a198edbfa2db5a43de567c9a548d2a2505b408befe24"
|
||||
);
|
||||
});
|
||||
|
||||
test("getCacheVersion with gzip compression does not change vesion", async () => {
|
||||
testUtils.setInput(Inputs.Path, "node_modules");
|
||||
const result = getCacheVersion(CompressionMethod.Gzip);
|
||||
|
||||
expect(result).toEqual(
|
||||
"b3e0c6cb5ecf32614eeb2997d905b9c297046d7cbf69062698f25b14b4cb0985"
|
||||
);
|
||||
});
|
||||
|
||||
test("getCacheVersion with no input throws", async () => {
|
||||
expect(() => getCacheVersion()).toThrow();
|
||||
});
|
||||
|
||||
interface TestResponse {
|
||||
statusCode: number;
|
||||
result: string | null;
|
||||
}
|
||||
|
||||
function handleResponse(
|
||||
response: TestResponse | undefined
|
||||
): Promise<TestResponse> {
|
||||
if (!response) {
|
||||
fail("Retry method called too many times");
|
||||
}
|
||||
|
||||
if (response.statusCode === 999) {
|
||||
throw Error("Test Error");
|
||||
} else {
|
||||
return Promise.resolve(response);
|
||||
}
|
||||
}
|
||||
|
||||
async function testRetryExpectingResult(
|
||||
responses: Array<TestResponse>,
|
||||
expectedResult: string | null
|
||||
): Promise<void> {
|
||||
responses = responses.reverse(); // Reverse responses since we pop from end
|
||||
|
||||
const actualResult = await retry(
|
||||
"test",
|
||||
() => handleResponse(responses.pop()),
|
||||
(response: TestResponse) => response.statusCode
|
||||
);
|
||||
|
||||
expect(actualResult.result).toEqual(expectedResult);
|
||||
}
|
||||
|
||||
async function testRetryExpectingError(
|
||||
responses: Array<TestResponse>
|
||||
): Promise<void> {
|
||||
responses = responses.reverse(); // Reverse responses since we pop from end
|
||||
|
||||
expect(
|
||||
retry(
|
||||
"test",
|
||||
() => handleResponse(responses.pop()),
|
||||
(response: TestResponse) => response.statusCode
|
||||
)
|
||||
).rejects.toBeInstanceOf(Error);
|
||||
}
|
||||
|
||||
test("retry works on successful response", async () => {
|
||||
await testRetryExpectingResult(
|
||||
[
|
||||
{
|
||||
statusCode: 200,
|
||||
result: "Ok"
|
||||
}
|
||||
],
|
||||
"Ok"
|
||||
);
|
||||
});
|
||||
|
||||
test("retry works after retryable status code", async () => {
|
||||
await testRetryExpectingResult(
|
||||
[
|
||||
{
|
||||
statusCode: 503,
|
||||
result: null
|
||||
},
|
||||
{
|
||||
statusCode: 200,
|
||||
result: "Ok"
|
||||
}
|
||||
],
|
||||
"Ok"
|
||||
);
|
||||
});
|
||||
|
||||
test("retry fails after exhausting retries", async () => {
|
||||
await testRetryExpectingError([
|
||||
{
|
||||
statusCode: 503,
|
||||
result: null
|
||||
},
|
||||
{
|
||||
statusCode: 503,
|
||||
result: null
|
||||
},
|
||||
{
|
||||
statusCode: 200,
|
||||
result: "Ok"
|
||||
}
|
||||
]);
|
||||
});
|
||||
|
||||
test("retry fails after non-retryable status code", async () => {
|
||||
await testRetryExpectingError([
|
||||
{
|
||||
statusCode: 500,
|
||||
result: null
|
||||
},
|
||||
{
|
||||
statusCode: 200,
|
||||
result: "Ok"
|
||||
}
|
||||
]);
|
||||
});
|
||||
|
||||
test("retry works after error", async () => {
|
||||
await testRetryExpectingResult(
|
||||
[
|
||||
{
|
||||
statusCode: 999,
|
||||
result: null
|
||||
},
|
||||
{
|
||||
statusCode: 200,
|
||||
result: "Ok"
|
||||
}
|
||||
],
|
||||
"Ok"
|
||||
);
|
||||
});
|
||||
|
||||
test("retry returns after client error", async () => {
|
||||
await testRetryExpectingResult(
|
||||
[
|
||||
{
|
||||
statusCode: 400,
|
||||
result: null
|
||||
},
|
||||
{
|
||||
statusCode: 200,
|
||||
result: "Ok"
|
||||
}
|
||||
],
|
||||
null
|
||||
);
|
||||
});
|
||||
@ -1,204 +0,0 @@
|
||||
import * as exec from "@actions/exec";
|
||||
import * as io from "@actions/io";
|
||||
import * as path from "path";
|
||||
|
||||
import { CacheFilename, CompressionMethod } from "../src/constants";
|
||||
import * as tar from "../src/tar";
|
||||
import * as utils from "../src/utils/actionUtils";
|
||||
|
||||
import fs = require("fs");
|
||||
|
||||
jest.mock("@actions/exec");
|
||||
jest.mock("@actions/io");
|
||||
|
||||
const IS_WINDOWS = process.platform === "win32";
|
||||
|
||||
function getTempDir(): string {
|
||||
return path.join(__dirname, "_temp", "tar");
|
||||
}
|
||||
|
||||
beforeAll(async () => {
|
||||
jest.spyOn(io, "which").mockImplementation(tool => {
|
||||
return Promise.resolve(tool);
|
||||
});
|
||||
|
||||
process.env["GITHUB_WORKSPACE"] = process.cwd();
|
||||
await jest.requireActual("@actions/io").rmRF(getTempDir());
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
delete process.env["GITHUB_WORKSPACE"];
|
||||
await jest.requireActual("@actions/io").rmRF(getTempDir());
|
||||
});
|
||||
|
||||
test("zstd extract tar", async () => {
|
||||
const mkdirMock = jest.spyOn(io, "mkdirP");
|
||||
const execMock = jest.spyOn(exec, "exec");
|
||||
|
||||
const archivePath = IS_WINDOWS
|
||||
? `${process.env["windir"]}\\fakepath\\cache.tar`
|
||||
: "cache.tar";
|
||||
const workspace = process.env["GITHUB_WORKSPACE"];
|
||||
|
||||
await tar.extractTar(archivePath, CompressionMethod.Zstd);
|
||||
|
||||
expect(mkdirMock).toHaveBeenCalledWith(workspace);
|
||||
const tarPath = IS_WINDOWS
|
||||
? `${process.env["windir"]}\\System32\\tar.exe`
|
||||
: "tar";
|
||||
expect(execMock).toHaveBeenCalledTimes(1);
|
||||
expect(execMock).toHaveBeenCalledWith(
|
||||
`"${tarPath}"`,
|
||||
[
|
||||
"--use-compress-program",
|
||||
"zstd -d --long=30",
|
||||
"-xf",
|
||||
IS_WINDOWS ? archivePath.replace(/\\/g, "/") : archivePath,
|
||||
"-P",
|
||||
"-C",
|
||||
IS_WINDOWS ? workspace?.replace(/\\/g, "/") : workspace
|
||||
],
|
||||
{ cwd: undefined }
|
||||
);
|
||||
});
|
||||
|
||||
test("gzip extract tar", async () => {
|
||||
const mkdirMock = jest.spyOn(io, "mkdirP");
|
||||
const execMock = jest.spyOn(exec, "exec");
|
||||
const archivePath = IS_WINDOWS
|
||||
? `${process.env["windir"]}\\fakepath\\cache.tar`
|
||||
: "cache.tar";
|
||||
const workspace = process.env["GITHUB_WORKSPACE"];
|
||||
|
||||
await tar.extractTar(archivePath, CompressionMethod.Gzip);
|
||||
|
||||
expect(mkdirMock).toHaveBeenCalledWith(workspace);
|
||||
const tarPath = IS_WINDOWS
|
||||
? `${process.env["windir"]}\\System32\\tar.exe`
|
||||
: "tar";
|
||||
expect(execMock).toHaveBeenCalledTimes(1);
|
||||
expect(execMock).toHaveBeenCalledWith(
|
||||
`"${tarPath}"`,
|
||||
[
|
||||
"-z",
|
||||
"-xf",
|
||||
IS_WINDOWS ? archivePath.replace(/\\/g, "/") : archivePath,
|
||||
"-P",
|
||||
"-C",
|
||||
IS_WINDOWS ? workspace?.replace(/\\/g, "/") : workspace
|
||||
],
|
||||
{ cwd: undefined }
|
||||
);
|
||||
});
|
||||
|
||||
test("gzip extract GNU tar on windows", async () => {
|
||||
if (IS_WINDOWS) {
|
||||
jest.spyOn(fs, "existsSync").mockReturnValueOnce(false);
|
||||
|
||||
const isGnuMock = jest
|
||||
.spyOn(utils, "useGnuTar")
|
||||
.mockReturnValue(Promise.resolve(true));
|
||||
const execMock = jest.spyOn(exec, "exec");
|
||||
const archivePath = `${process.env["windir"]}\\fakepath\\cache.tar`;
|
||||
const workspace = process.env["GITHUB_WORKSPACE"];
|
||||
|
||||
await tar.extractTar(archivePath, CompressionMethod.Gzip);
|
||||
|
||||
expect(isGnuMock).toHaveBeenCalledTimes(1);
|
||||
expect(execMock).toHaveBeenCalledTimes(1);
|
||||
expect(execMock).toHaveBeenCalledWith(
|
||||
`"tar"`,
|
||||
[
|
||||
"-z",
|
||||
"-xf",
|
||||
archivePath.replace(/\\/g, "/"),
|
||||
"-P",
|
||||
"-C",
|
||||
workspace?.replace(/\\/g, "/"),
|
||||
"--force-local"
|
||||
],
|
||||
{ cwd: undefined }
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
test("zstd create tar", async () => {
|
||||
const execMock = jest.spyOn(exec, "exec");
|
||||
|
||||
const archiveFolder = getTempDir();
|
||||
const workspace = process.env["GITHUB_WORKSPACE"];
|
||||
const sourceDirectories = ["~/.npm/cache", `${workspace}/dist`];
|
||||
|
||||
await fs.promises.mkdir(archiveFolder, { recursive: true });
|
||||
|
||||
await tar.createTar(
|
||||
archiveFolder,
|
||||
sourceDirectories,
|
||||
CompressionMethod.Zstd
|
||||
);
|
||||
|
||||
const tarPath = IS_WINDOWS
|
||||
? `${process.env["windir"]}\\System32\\tar.exe`
|
||||
: "tar";
|
||||
|
||||
expect(execMock).toHaveBeenCalledTimes(1);
|
||||
expect(execMock).toHaveBeenCalledWith(
|
||||
`"${tarPath}"`,
|
||||
[
|
||||
"--use-compress-program",
|
||||
"zstd -T0 --long=30",
|
||||
"-cf",
|
||||
IS_WINDOWS
|
||||
? CacheFilename.Zstd.replace(/\\/g, "/")
|
||||
: CacheFilename.Zstd,
|
||||
"-P",
|
||||
"-C",
|
||||
IS_WINDOWS ? workspace?.replace(/\\/g, "/") : workspace,
|
||||
"--files-from",
|
||||
"manifest.txt"
|
||||
],
|
||||
{
|
||||
cwd: archiveFolder
|
||||
}
|
||||
);
|
||||
});
|
||||
|
||||
test("gzip create tar", async () => {
|
||||
const execMock = jest.spyOn(exec, "exec");
|
||||
|
||||
const archiveFolder = getTempDir();
|
||||
const workspace = process.env["GITHUB_WORKSPACE"];
|
||||
const sourceDirectories = ["~/.npm/cache", `${workspace}/dist`];
|
||||
|
||||
await fs.promises.mkdir(archiveFolder, { recursive: true });
|
||||
|
||||
await tar.createTar(
|
||||
archiveFolder,
|
||||
sourceDirectories,
|
||||
CompressionMethod.Gzip
|
||||
);
|
||||
|
||||
const tarPath = IS_WINDOWS
|
||||
? `${process.env["windir"]}\\System32\\tar.exe`
|
||||
: "tar";
|
||||
|
||||
expect(execMock).toHaveBeenCalledTimes(1);
|
||||
expect(execMock).toHaveBeenCalledWith(
|
||||
`"${tarPath}"`,
|
||||
[
|
||||
"-z",
|
||||
"-cf",
|
||||
IS_WINDOWS
|
||||
? CacheFilename.Gzip.replace(/\\/g, "/")
|
||||
: CacheFilename.Gzip,
|
||||
"-P",
|
||||
"-C",
|
||||
IS_WINDOWS ? workspace?.replace(/\\/g, "/") : workspace,
|
||||
"--files-from",
|
||||
"manifest.txt"
|
||||
],
|
||||
{
|
||||
cwd: archiveFolder
|
||||
}
|
||||
);
|
||||
});
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@ -1,424 +0,0 @@
|
||||
import * as core from "@actions/core";
|
||||
import { HttpClient, HttpCodes } from "@actions/http-client";
|
||||
import { BearerCredentialHandler } from "@actions/http-client/auth";
|
||||
import {
|
||||
IHttpClientResponse,
|
||||
IRequestOptions,
|
||||
ITypedResponse
|
||||
} from "@actions/http-client/interfaces";
|
||||
import * as crypto from "crypto";
|
||||
import * as fs from "fs";
|
||||
import * as stream from "stream";
|
||||
import * as util from "util";
|
||||
|
||||
import { CompressionMethod, Inputs, SocketTimeout } from "./constants";
|
||||
import {
|
||||
ArtifactCacheEntry,
|
||||
CacheOptions,
|
||||
CommitCacheRequest,
|
||||
ReserveCacheRequest,
|
||||
ReserveCacheResponse
|
||||
} from "./contracts";
|
||||
import * as utils from "./utils/actionUtils";
|
||||
|
||||
const versionSalt = "1.0";
|
||||
|
||||
function isSuccessStatusCode(statusCode?: number): boolean {
|
||||
if (!statusCode) {
|
||||
return false;
|
||||
}
|
||||
return statusCode >= 200 && statusCode < 300;
|
||||
}
|
||||
|
||||
function isServerErrorStatusCode(statusCode?: number): boolean {
|
||||
if (!statusCode) {
|
||||
return true;
|
||||
}
|
||||
return statusCode >= 500;
|
||||
}
|
||||
|
||||
function isRetryableStatusCode(statusCode?: number): boolean {
|
||||
if (!statusCode) {
|
||||
return false;
|
||||
}
|
||||
const retryableStatusCodes = [
|
||||
HttpCodes.BadGateway,
|
||||
HttpCodes.ServiceUnavailable,
|
||||
HttpCodes.GatewayTimeout
|
||||
];
|
||||
return retryableStatusCodes.includes(statusCode);
|
||||
}
|
||||
|
||||
function getCacheApiUrl(resource: string): string {
|
||||
// Ideally we just use ACTIONS_CACHE_URL
|
||||
const baseUrl: string = (
|
||||
process.env["ACTIONS_CACHE_URL"] ||
|
||||
process.env["ACTIONS_RUNTIME_URL"] ||
|
||||
""
|
||||
).replace("pipelines", "artifactcache");
|
||||
if (!baseUrl) {
|
||||
throw new Error(
|
||||
"Cache Service Url not found, unable to restore cache."
|
||||
);
|
||||
}
|
||||
|
||||
const url = `${baseUrl}_apis/artifactcache/${resource}`;
|
||||
core.debug(`Resource Url: ${url}`);
|
||||
return url;
|
||||
}
|
||||
|
||||
function createAcceptHeader(type: string, apiVersion: string): string {
|
||||
return `${type};api-version=${apiVersion}`;
|
||||
}
|
||||
|
||||
function getRequestOptions(): IRequestOptions {
|
||||
const requestOptions: IRequestOptions = {
|
||||
headers: {
|
||||
Accept: createAcceptHeader("application/json", "6.0-preview.1")
|
||||
}
|
||||
};
|
||||
|
||||
return requestOptions;
|
||||
}
|
||||
|
||||
function createHttpClient(): HttpClient {
|
||||
const token = process.env["ACTIONS_RUNTIME_TOKEN"] || "";
|
||||
const bearerCredentialHandler = new BearerCredentialHandler(token);
|
||||
|
||||
return new HttpClient(
|
||||
"actions/cache",
|
||||
[bearerCredentialHandler],
|
||||
getRequestOptions()
|
||||
);
|
||||
}
|
||||
|
||||
export function getCacheVersion(compressionMethod?: CompressionMethod): string {
|
||||
const components = [core.getInput(Inputs.Path, { required: true })].concat(
|
||||
compressionMethod == CompressionMethod.Zstd ? [compressionMethod] : []
|
||||
);
|
||||
|
||||
// Add salt to cache version to support breaking changes in cache entry
|
||||
components.push(versionSalt);
|
||||
|
||||
return crypto
|
||||
.createHash("sha256")
|
||||
.update(components.join("|"))
|
||||
.digest("hex");
|
||||
}
|
||||
|
||||
export async function retry<T>(
|
||||
name: string,
|
||||
method: () => Promise<T>,
|
||||
getStatusCode: (T) => number | undefined,
|
||||
maxAttempts = 2
|
||||
): Promise<T> {
|
||||
let response: T | undefined = undefined;
|
||||
let statusCode: number | undefined = undefined;
|
||||
let isRetryable = false;
|
||||
let errorMessage = "";
|
||||
let attempt = 1;
|
||||
|
||||
while (attempt <= maxAttempts) {
|
||||
try {
|
||||
response = await method();
|
||||
statusCode = getStatusCode(response);
|
||||
|
||||
if (!isServerErrorStatusCode(statusCode)) {
|
||||
return response;
|
||||
}
|
||||
|
||||
isRetryable = isRetryableStatusCode(statusCode);
|
||||
errorMessage = `Cache service responded with ${statusCode}`;
|
||||
} catch (error) {
|
||||
isRetryable = true;
|
||||
errorMessage = error.message;
|
||||
}
|
||||
|
||||
core.debug(
|
||||
`${name} - Attempt ${attempt} of ${maxAttempts} failed with error: ${errorMessage}`
|
||||
);
|
||||
|
||||
if (!isRetryable) {
|
||||
core.debug(`${name} - Error is not retryable`);
|
||||
break;
|
||||
}
|
||||
|
||||
attempt++;
|
||||
}
|
||||
|
||||
throw Error(`${name} failed: ${errorMessage}`);
|
||||
}
|
||||
|
||||
export async function retryTypedResponse<T>(
|
||||
name: string,
|
||||
method: () => Promise<ITypedResponse<T>>,
|
||||
maxAttempts = 2
|
||||
): Promise<ITypedResponse<T>> {
|
||||
return await retry(
|
||||
name,
|
||||
method,
|
||||
(response: ITypedResponse<T>) => response.statusCode,
|
||||
maxAttempts
|
||||
);
|
||||
}
|
||||
|
||||
export async function retryHttpClientResponse<T>(
|
||||
name: string,
|
||||
method: () => Promise<IHttpClientResponse>,
|
||||
maxAttempts = 2
|
||||
): Promise<IHttpClientResponse> {
|
||||
return await retry(
|
||||
name,
|
||||
method,
|
||||
(response: IHttpClientResponse) => response.message.statusCode,
|
||||
maxAttempts
|
||||
);
|
||||
}
|
||||
|
||||
export async function getCacheEntry(
|
||||
keys: string[],
|
||||
options?: CacheOptions
|
||||
): Promise<ArtifactCacheEntry | null> {
|
||||
const httpClient = createHttpClient();
|
||||
const version = getCacheVersion(options?.compressionMethod);
|
||||
const resource = `cache?keys=${encodeURIComponent(
|
||||
keys.join(",")
|
||||
)}&version=${version}`;
|
||||
|
||||
const response = await retryTypedResponse("getCacheEntry", () =>
|
||||
httpClient.getJson<ArtifactCacheEntry>(getCacheApiUrl(resource))
|
||||
);
|
||||
if (response.statusCode === 204) {
|
||||
return null;
|
||||
}
|
||||
if (!isSuccessStatusCode(response.statusCode)) {
|
||||
throw new Error(`Cache service responded with ${response.statusCode}`);
|
||||
}
|
||||
|
||||
const cacheResult = response.result;
|
||||
const cacheDownloadUrl = cacheResult?.archiveLocation;
|
||||
if (!cacheDownloadUrl) {
|
||||
throw new Error("Cache not found.");
|
||||
}
|
||||
core.setSecret(cacheDownloadUrl);
|
||||
core.debug(`Cache Result:`);
|
||||
core.debug(JSON.stringify(cacheResult));
|
||||
|
||||
return cacheResult;
|
||||
}
|
||||
|
||||
async function pipeResponseToStream(
|
||||
response: IHttpClientResponse,
|
||||
output: NodeJS.WritableStream
|
||||
): Promise<void> {
|
||||
const pipeline = util.promisify(stream.pipeline);
|
||||
await pipeline(response.message, output);
|
||||
}
|
||||
|
||||
export async function downloadCache(
|
||||
archiveLocation: string,
|
||||
archivePath: string
|
||||
): Promise<void> {
|
||||
const stream = fs.createWriteStream(archivePath);
|
||||
const httpClient = new HttpClient("actions/cache");
|
||||
const downloadResponse = await retryHttpClientResponse(
|
||||
"downloadCache",
|
||||
() => httpClient.get(archiveLocation)
|
||||
);
|
||||
|
||||
// Abort download if no traffic received over the socket.
|
||||
downloadResponse.message.socket.setTimeout(SocketTimeout, () => {
|
||||
downloadResponse.message.destroy();
|
||||
core.debug(
|
||||
`Aborting download, socket timed out after ${SocketTimeout} ms`
|
||||
);
|
||||
});
|
||||
|
||||
await pipeResponseToStream(downloadResponse, stream);
|
||||
|
||||
// Validate download size.
|
||||
const contentLengthHeader =
|
||||
downloadResponse.message.headers["content-length"];
|
||||
|
||||
if (contentLengthHeader) {
|
||||
const expectedLength = parseInt(contentLengthHeader);
|
||||
const actualLength = utils.getArchiveFileSize(archivePath);
|
||||
|
||||
if (actualLength != expectedLength) {
|
||||
throw new Error(
|
||||
`Incomplete download. Expected file size: ${expectedLength}, actual file size: ${actualLength}`
|
||||
);
|
||||
}
|
||||
} else {
|
||||
core.debug("Unable to validate download, no Content-Length header");
|
||||
}
|
||||
}
|
||||
|
||||
// Reserve Cache
|
||||
export async function reserveCache(
|
||||
key: string,
|
||||
options?: CacheOptions
|
||||
): Promise<number> {
|
||||
const httpClient = createHttpClient();
|
||||
const version = getCacheVersion(options?.compressionMethod);
|
||||
|
||||
const reserveCacheRequest: ReserveCacheRequest = {
|
||||
key,
|
||||
version
|
||||
};
|
||||
const response = await retryTypedResponse("reserveCache", () =>
|
||||
httpClient.postJson<ReserveCacheResponse>(
|
||||
getCacheApiUrl("caches"),
|
||||
reserveCacheRequest
|
||||
)
|
||||
);
|
||||
return response?.result?.cacheId ?? -1;
|
||||
}
|
||||
|
||||
function getContentRange(start: number, end: number): string {
|
||||
// Format: `bytes start-end/filesize
|
||||
// start and end are inclusive
|
||||
// filesize can be *
|
||||
// For a 200 byte chunk starting at byte 0:
|
||||
// Content-Range: bytes 0-199/*
|
||||
return `bytes ${start}-${end}/*`;
|
||||
}
|
||||
|
||||
async function uploadChunk(
|
||||
httpClient: HttpClient,
|
||||
resourceUrl: string,
|
||||
openStream: () => NodeJS.ReadableStream,
|
||||
start: number,
|
||||
end: number
|
||||
): Promise<void> {
|
||||
core.debug(
|
||||
`Uploading chunk of size ${end -
|
||||
start +
|
||||
1} bytes at offset ${start} with content range: ${getContentRange(
|
||||
start,
|
||||
end
|
||||
)}`
|
||||
);
|
||||
const additionalHeaders = {
|
||||
"Content-Type": "application/octet-stream",
|
||||
"Content-Range": getContentRange(start, end)
|
||||
};
|
||||
|
||||
await retryHttpClientResponse(
|
||||
`uploadChunk (start: ${start}, end: ${end})`,
|
||||
() =>
|
||||
httpClient.sendStream(
|
||||
"PATCH",
|
||||
resourceUrl,
|
||||
openStream(),
|
||||
additionalHeaders
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
function parseEnvNumber(key: string): number | undefined {
|
||||
const value = Number(process.env[key]);
|
||||
if (Number.isNaN(value) || value < 0) {
|
||||
return undefined;
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
async function uploadFile(
|
||||
httpClient: HttpClient,
|
||||
cacheId: number,
|
||||
archivePath: string
|
||||
): Promise<void> {
|
||||
// Upload Chunks
|
||||
const fileSize = fs.statSync(archivePath).size;
|
||||
const resourceUrl = getCacheApiUrl(`caches/${cacheId.toString()}`);
|
||||
const fd = fs.openSync(archivePath, "r");
|
||||
|
||||
const concurrency = parseEnvNumber("CACHE_UPLOAD_CONCURRENCY") ?? 4; // # of HTTP requests in parallel
|
||||
const MAX_CHUNK_SIZE =
|
||||
parseEnvNumber("CACHE_UPLOAD_CHUNK_SIZE") ?? 32 * 1024 * 1024; // 32 MB Chunks
|
||||
core.debug(`Concurrency: ${concurrency} and Chunk Size: ${MAX_CHUNK_SIZE}`);
|
||||
|
||||
const parallelUploads = [...new Array(concurrency).keys()];
|
||||
core.debug("Awaiting all uploads");
|
||||
let offset = 0;
|
||||
|
||||
try {
|
||||
await Promise.all(
|
||||
parallelUploads.map(async () => {
|
||||
while (offset < fileSize) {
|
||||
const chunkSize = Math.min(
|
||||
fileSize - offset,
|
||||
MAX_CHUNK_SIZE
|
||||
);
|
||||
const start = offset;
|
||||
const end = offset + chunkSize - 1;
|
||||
offset += MAX_CHUNK_SIZE;
|
||||
|
||||
await uploadChunk(
|
||||
httpClient,
|
||||
resourceUrl,
|
||||
() =>
|
||||
fs
|
||||
.createReadStream(archivePath, {
|
||||
fd,
|
||||
start,
|
||||
end,
|
||||
autoClose: false
|
||||
})
|
||||
.on("error", error => {
|
||||
throw new Error(
|
||||
`Cache upload failed because file read failed with ${error.Message}`
|
||||
);
|
||||
}),
|
||||
start,
|
||||
end
|
||||
);
|
||||
}
|
||||
})
|
||||
);
|
||||
} finally {
|
||||
fs.closeSync(fd);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
async function commitCache(
|
||||
httpClient: HttpClient,
|
||||
cacheId: number,
|
||||
filesize: number
|
||||
): Promise<ITypedResponse<null>> {
|
||||
const commitCacheRequest: CommitCacheRequest = { size: filesize };
|
||||
return await retryTypedResponse("commitCache", () =>
|
||||
httpClient.postJson<null>(
|
||||
getCacheApiUrl(`caches/${cacheId.toString()}`),
|
||||
commitCacheRequest
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
export async function saveCache(
|
||||
cacheId: number,
|
||||
archivePath: string
|
||||
): Promise<void> {
|
||||
const httpClient = createHttpClient();
|
||||
|
||||
core.debug("Upload cache");
|
||||
await uploadFile(httpClient, cacheId, archivePath);
|
||||
|
||||
// Commit Cache
|
||||
core.debug("Commiting cache");
|
||||
const cacheSize = utils.getArchiveFileSize(archivePath);
|
||||
const commitCacheResponse = await commitCache(
|
||||
httpClient,
|
||||
cacheId,
|
||||
cacheSize
|
||||
);
|
||||
if (!isSuccessStatusCode(commitCacheResponse.statusCode)) {
|
||||
throw new Error(
|
||||
`Cache service responded with ${commitCacheResponse.statusCode} during commit cache.`
|
||||
);
|
||||
}
|
||||
|
||||
core.info("Cache saved successfully");
|
||||
}
|
||||
@ -1,25 +0,0 @@
|
||||
import { CompressionMethod } from "./constants";
|
||||
|
||||
export interface ArtifactCacheEntry {
|
||||
cacheKey?: string;
|
||||
scope?: string;
|
||||
creationTime?: string;
|
||||
archiveLocation?: string;
|
||||
}
|
||||
|
||||
export interface CommitCacheRequest {
|
||||
size: number;
|
||||
}
|
||||
|
||||
export interface ReserveCacheRequest {
|
||||
key: string;
|
||||
version?: string;
|
||||
}
|
||||
|
||||
export interface ReserveCacheResponse {
|
||||
cacheId: number;
|
||||
}
|
||||
|
||||
export interface CacheOptions {
|
||||
compressionMethod?: CompressionMethod;
|
||||
}
|
||||
@ -1,87 +0,0 @@
|
||||
import { exec } from "@actions/exec";
|
||||
import * as io from "@actions/io";
|
||||
import { existsSync, writeFileSync } from "fs";
|
||||
import * as path from "path";
|
||||
|
||||
import { CompressionMethod } from "./constants";
|
||||
import * as utils from "./utils/actionUtils";
|
||||
|
||||
async function getTarPath(args: string[]): Promise<string> {
|
||||
// Explicitly use BSD Tar on Windows
|
||||
const IS_WINDOWS = process.platform === "win32";
|
||||
if (IS_WINDOWS) {
|
||||
const systemTar = `${process.env["windir"]}\\System32\\tar.exe`;
|
||||
if (existsSync(systemTar)) {
|
||||
return systemTar;
|
||||
} else if (await utils.useGnuTar()) {
|
||||
args.push("--force-local");
|
||||
}
|
||||
}
|
||||
return await io.which("tar", true);
|
||||
}
|
||||
|
||||
async function execTar(args: string[], cwd?: string): Promise<void> {
|
||||
try {
|
||||
await exec(`"${await getTarPath(args)}"`, args, { cwd: cwd });
|
||||
} catch (error) {
|
||||
throw new Error(`Tar failed with error: ${error?.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
function getWorkingDirectory(): string {
|
||||
return process.env["GITHUB_WORKSPACE"] ?? process.cwd();
|
||||
}
|
||||
|
||||
export async function extractTar(
|
||||
archivePath: string,
|
||||
compressionMethod: CompressionMethod
|
||||
): Promise<void> {
|
||||
// Create directory to extract tar into
|
||||
const workingDirectory = getWorkingDirectory();
|
||||
await io.mkdirP(workingDirectory);
|
||||
// --d: Decompress.
|
||||
// --long=#: Enables long distance matching with # bits. Maximum is 30 (1GB) on 32-bit OS and 31 (2GB) on 64-bit.
|
||||
// Using 30 here because we also support 32-bit self-hosted runners.
|
||||
const args = [
|
||||
...(compressionMethod == CompressionMethod.Zstd
|
||||
? ["--use-compress-program", "zstd -d --long=30"]
|
||||
: ["-z"]),
|
||||
"-xf",
|
||||
archivePath.replace(new RegExp("\\" + path.sep, "g"), "/"),
|
||||
"-P",
|
||||
"-C",
|
||||
workingDirectory.replace(new RegExp("\\" + path.sep, "g"), "/")
|
||||
];
|
||||
await execTar(args);
|
||||
}
|
||||
|
||||
export async function createTar(
|
||||
archiveFolder: string,
|
||||
sourceDirectories: string[],
|
||||
compressionMethod: CompressionMethod
|
||||
): Promise<void> {
|
||||
// Write source directories to manifest.txt to avoid command length limits
|
||||
const manifestFilename = "manifest.txt";
|
||||
const cacheFileName = utils.getCacheFileName(compressionMethod);
|
||||
writeFileSync(
|
||||
path.join(archiveFolder, manifestFilename),
|
||||
sourceDirectories.join("\n")
|
||||
);
|
||||
// -T#: Compress using # working thread. If # is 0, attempt to detect and use the number of physical CPU cores.
|
||||
// --long=#: Enables long distance matching with # bits. Maximum is 30 (1GB) on 32-bit OS and 31 (2GB) on 64-bit.
|
||||
// Using 30 here because we also support 32-bit self-hosted runners.
|
||||
const workingDirectory = getWorkingDirectory();
|
||||
const args = [
|
||||
...(compressionMethod == CompressionMethod.Zstd
|
||||
? ["--use-compress-program", "zstd -T0 --long=30"]
|
||||
: ["-z"]),
|
||||
"-cf",
|
||||
cacheFileName.replace(new RegExp("\\" + path.sep, "g"), "/"),
|
||||
"-P",
|
||||
"-C",
|
||||
workingDirectory.replace(new RegExp("\\" + path.sep, "g"), "/"),
|
||||
"--files-from",
|
||||
manifestFilename
|
||||
];
|
||||
await execTar(args, archiveFolder);
|
||||
}
|
||||
Loading…
Reference in New Issue