mirror of https://github.com/actions/cache.git
Boris Staal
3 years ago
committed by
GitHub
commit
1a3e6024cc
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,253 @@
|
|||||||
|
import * as core from "@actions/core";
|
||||||
|
import { S3ClientConfig } from "@aws-sdk/client-s3";
|
||||||
|
import * as path from "path";
|
||||||
|
|
||||||
|
import * as utils from "./utils/cacheUtils";
|
||||||
|
import * as cacheHttpClient from "./utils/clientUtils";
|
||||||
|
import { DownloadOptions, UploadOptions } from "./utils/contracts";
|
||||||
|
import { createTar, extractTar, listTar } from "./utils/tar";
|
||||||
|
|
||||||
|
export class ValidationError extends Error {
|
||||||
|
constructor(message: string) {
|
||||||
|
super(message);
|
||||||
|
this.name = "ValidationError";
|
||||||
|
Object.setPrototypeOf(this, ValidationError.prototype);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export class ReserveCacheError extends Error {
|
||||||
|
constructor(message: string) {
|
||||||
|
super(message);
|
||||||
|
this.name = "ReserveCacheError";
|
||||||
|
Object.setPrototypeOf(this, ReserveCacheError.prototype);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function checkPaths(paths: string[]): void {
|
||||||
|
if (!paths || paths.length === 0) {
|
||||||
|
throw new ValidationError(
|
||||||
|
`Path Validation Error: At least one directory or file path is required`
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function checkKey(key: string): void {
|
||||||
|
if (key.length > 512) {
|
||||||
|
throw new ValidationError(
|
||||||
|
`Key Validation Error: ${key} cannot be larger than 512 characters.`
|
||||||
|
);
|
||||||
|
}
|
||||||
|
const regex = /^[^,]*$/;
|
||||||
|
if (!regex.test(key)) {
|
||||||
|
throw new ValidationError(
|
||||||
|
`Key Validation Error: ${key} cannot contain commas.`
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* isFeatureAvailable to check the presence of Actions cache service
|
||||||
|
*
|
||||||
|
* @returns boolean return true if Actions cache service feature is available, otherwise false
|
||||||
|
*/
|
||||||
|
|
||||||
|
export function isFeatureAvailable(): boolean {
|
||||||
|
return !!process.env["ACTIONS_CACHE_URL"];
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Restores cache from keys
|
||||||
|
*
|
||||||
|
* @param paths a list of file paths to restore from the cache
|
||||||
|
* @param primaryKey an explicit key for restoring the cache
|
||||||
|
* @param restoreKeys an optional ordered list of keys to use for restoring the cache if no cache hit occurred for key
|
||||||
|
* @param options cache download options
|
||||||
|
* @param s3Options upload options for AWS S3
|
||||||
|
* @param s3BucketName a name of AWS S3 bucket
|
||||||
|
* @returns string returns the key for the cache hit, otherwise returns undefined
|
||||||
|
*/
|
||||||
|
export async function restoreCache(
|
||||||
|
paths: string[],
|
||||||
|
primaryKey: string,
|
||||||
|
restoreKeys?: string[],
|
||||||
|
options?: DownloadOptions,
|
||||||
|
s3Options?: S3ClientConfig,
|
||||||
|
s3BucketName?: string
|
||||||
|
): Promise<string | undefined> {
|
||||||
|
checkPaths(paths);
|
||||||
|
|
||||||
|
restoreKeys = restoreKeys || [];
|
||||||
|
const keys = [primaryKey, ...restoreKeys];
|
||||||
|
|
||||||
|
core.debug("Resolved Keys:");
|
||||||
|
core.debug(JSON.stringify(keys));
|
||||||
|
|
||||||
|
if (keys.length > 10) {
|
||||||
|
throw new ValidationError(
|
||||||
|
`Key Validation Error: Keys are limited to a maximum of 10.`
|
||||||
|
);
|
||||||
|
}
|
||||||
|
for (const key of keys) {
|
||||||
|
checkKey(key);
|
||||||
|
}
|
||||||
|
|
||||||
|
const compressionMethod = await utils.getCompressionMethod();
|
||||||
|
|
||||||
|
// path are needed to compute version
|
||||||
|
const cacheEntry = await cacheHttpClient.getCacheEntry(
|
||||||
|
keys,
|
||||||
|
paths,
|
||||||
|
{
|
||||||
|
compressionMethod
|
||||||
|
},
|
||||||
|
s3Options,
|
||||||
|
s3BucketName
|
||||||
|
);
|
||||||
|
if (!cacheEntry || (!cacheEntry.archiveLocation && !cacheEntry.cacheKey)) {
|
||||||
|
// Cache not found
|
||||||
|
return undefined;
|
||||||
|
}
|
||||||
|
|
||||||
|
const archivePath = path.join(
|
||||||
|
await utils.createTempDirectory(),
|
||||||
|
utils.getCacheFileName(compressionMethod)
|
||||||
|
);
|
||||||
|
core.debug(`Archive Path: ${archivePath}`);
|
||||||
|
|
||||||
|
try {
|
||||||
|
// Download the cache from the cache entry
|
||||||
|
await cacheHttpClient.downloadCache(
|
||||||
|
cacheEntry,
|
||||||
|
archivePath,
|
||||||
|
options,
|
||||||
|
s3Options,
|
||||||
|
s3BucketName
|
||||||
|
);
|
||||||
|
|
||||||
|
if (core.isDebug()) {
|
||||||
|
await listTar(archivePath, compressionMethod);
|
||||||
|
}
|
||||||
|
|
||||||
|
const archiveFileSize = utils.getArchiveFileSizeInBytes(archivePath);
|
||||||
|
core.info(
|
||||||
|
`Cache Size: ~${Math.round(
|
||||||
|
archiveFileSize / (1024 * 1024)
|
||||||
|
)} MB (${archiveFileSize} B)`
|
||||||
|
);
|
||||||
|
|
||||||
|
await extractTar(archivePath, compressionMethod);
|
||||||
|
core.info("Cache restored successfully");
|
||||||
|
} finally {
|
||||||
|
// Try to delete the archive to save space
|
||||||
|
try {
|
||||||
|
await utils.unlinkFile(archivePath);
|
||||||
|
} catch (error) {
|
||||||
|
core.debug(`Failed to delete archive: ${error}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return cacheEntry.cacheKey;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Saves a list of files with the specified key
|
||||||
|
*
|
||||||
|
* @param paths a list of file paths to be cached
|
||||||
|
* @param key an explicit key for restoring the cache
|
||||||
|
* @param options cache upload options
|
||||||
|
* @param s3Options upload options for AWS S3
|
||||||
|
* @param s3BucketName a name of AWS S3 bucket
|
||||||
|
* @returns number returns cacheId if the cache was saved successfully and throws an error if save fails
|
||||||
|
*/
|
||||||
|
export async function saveCache(
|
||||||
|
paths: string[],
|
||||||
|
key: string,
|
||||||
|
options?: UploadOptions,
|
||||||
|
s3Options?: S3ClientConfig,
|
||||||
|
s3BucketName?: string
|
||||||
|
): Promise<number> {
|
||||||
|
checkPaths(paths);
|
||||||
|
checkKey(key);
|
||||||
|
|
||||||
|
const compressionMethod = await utils.getCompressionMethod();
|
||||||
|
let cacheId = 0;
|
||||||
|
|
||||||
|
const cachePaths = await utils.resolvePaths(paths);
|
||||||
|
core.debug("Cache Paths:");
|
||||||
|
core.debug(`${JSON.stringify(cachePaths)}`);
|
||||||
|
|
||||||
|
const archiveFolder = await utils.createTempDirectory();
|
||||||
|
const archivePath = path.join(
|
||||||
|
archiveFolder,
|
||||||
|
utils.getCacheFileName(compressionMethod)
|
||||||
|
);
|
||||||
|
|
||||||
|
core.debug(`Archive Path: ${archivePath}`);
|
||||||
|
|
||||||
|
try {
|
||||||
|
await createTar(archiveFolder, cachePaths, compressionMethod);
|
||||||
|
if (core.isDebug()) {
|
||||||
|
await listTar(archivePath, compressionMethod);
|
||||||
|
}
|
||||||
|
const fileSizeLimit = 10 * 1024 * 1024 * 1024; // 10GB per repo limit
|
||||||
|
const archiveFileSize = utils.getArchiveFileSizeInBytes(archivePath);
|
||||||
|
core.debug(`File Size: ${archiveFileSize}`);
|
||||||
|
|
||||||
|
// For GHES, this check will take place in ReserveCache API with enterprise file size limit
|
||||||
|
if (archiveFileSize > fileSizeLimit) {
|
||||||
|
throw new Error(
|
||||||
|
`Cache size of ~${Math.round(
|
||||||
|
archiveFileSize / (1024 * 1024)
|
||||||
|
)} MB (${archiveFileSize} B) is over the 10GB limit, not saving cache.`
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!(s3Options && s3BucketName)) {
|
||||||
|
core.debug("Reserving Cache");
|
||||||
|
const reserveCacheResponse = await cacheHttpClient.reserveCache(
|
||||||
|
key,
|
||||||
|
paths,
|
||||||
|
{
|
||||||
|
compressionMethod,
|
||||||
|
cacheSize: archiveFileSize
|
||||||
|
},
|
||||||
|
s3Options,
|
||||||
|
s3BucketName
|
||||||
|
);
|
||||||
|
|
||||||
|
if (reserveCacheResponse?.result?.cacheId) {
|
||||||
|
cacheId = reserveCacheResponse?.result?.cacheId;
|
||||||
|
} else if (reserveCacheResponse?.statusCode === 400) {
|
||||||
|
throw new Error(
|
||||||
|
reserveCacheResponse?.error?.message ??
|
||||||
|
`Cache size of ~${Math.round(
|
||||||
|
archiveFileSize / (1024 * 1024)
|
||||||
|
)} MB (${archiveFileSize} B) is over the data cap limit, not saving cache.`
|
||||||
|
);
|
||||||
|
} else {
|
||||||
|
throw new ReserveCacheError(
|
||||||
|
`Unable to reserve cache with key ${key}, another job may be creating this cache. More details: ${reserveCacheResponse?.error?.message}`
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
core.debug(`Saving Cache (ID: ${cacheId})`);
|
||||||
|
await cacheHttpClient.saveCache(
|
||||||
|
cacheId,
|
||||||
|
archivePath,
|
||||||
|
key,
|
||||||
|
options,
|
||||||
|
s3Options,
|
||||||
|
s3BucketName
|
||||||
|
);
|
||||||
|
} finally {
|
||||||
|
// Try to delete the archive to save space
|
||||||
|
try {
|
||||||
|
await utils.unlinkFile(archivePath);
|
||||||
|
} catch (error) {
|
||||||
|
core.debug(`Failed to delete archive: ${error}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return cacheId;
|
||||||
|
}
|
||||||
@ -0,0 +1,154 @@
|
|||||||
|
import * as core from "@actions/core";
|
||||||
|
import * as exec from "@actions/exec";
|
||||||
|
import * as glob from "@actions/glob";
|
||||||
|
import * as io from "@actions/io";
|
||||||
|
import * as fs from "fs";
|
||||||
|
import * as path from "path";
|
||||||
|
import * as semver from "semver";
|
||||||
|
import { v4 as uuidV4 } from "uuid";
|
||||||
|
|
||||||
|
export enum CacheFilename {
|
||||||
|
Gzip = "cache.tgz",
|
||||||
|
Zstd = "cache.tzst"
|
||||||
|
}
|
||||||
|
|
||||||
|
export enum CompressionMethod {
|
||||||
|
Gzip = "gzip",
|
||||||
|
// Long range mode was added to zstd in v1.3.2.
|
||||||
|
// This enum is for earlier version of zstd that does not have --long support
|
||||||
|
ZstdWithoutLong = "zstd-without-long",
|
||||||
|
Zstd = "zstd"
|
||||||
|
}
|
||||||
|
|
||||||
|
// The default number of retry attempts.
|
||||||
|
export const DefaultRetryAttempts = 2;
|
||||||
|
|
||||||
|
// The default delay in milliseconds between retry attempts.
|
||||||
|
export const DefaultRetryDelay = 5000;
|
||||||
|
|
||||||
|
// Socket timeout in milliseconds during download. If no traffic is received
|
||||||
|
// over the socket during this period, the socket is destroyed and the download
|
||||||
|
// is aborted.
|
||||||
|
export const SocketTimeout = 5000;
|
||||||
|
|
||||||
|
async function getVersion(app: string): Promise<string> {
|
||||||
|
core.debug(`Checking ${app} --version`);
|
||||||
|
let versionOutput = "";
|
||||||
|
try {
|
||||||
|
await exec.exec(`${app} --version`, [], {
|
||||||
|
ignoreReturnCode: true,
|
||||||
|
silent: true,
|
||||||
|
listeners: {
|
||||||
|
stdout: (data: Buffer): string =>
|
||||||
|
(versionOutput += data.toString()),
|
||||||
|
stderr: (data: Buffer): string =>
|
||||||
|
(versionOutput += data.toString())
|
||||||
|
}
|
||||||
|
});
|
||||||
|
} catch (err: any) {
|
||||||
|
core.debug(err.message);
|
||||||
|
}
|
||||||
|
|
||||||
|
versionOutput = versionOutput.trim();
|
||||||
|
core.debug(versionOutput);
|
||||||
|
return versionOutput;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function getArchiveFileSizeInBytes(filePath: string): number {
|
||||||
|
return fs.statSync(filePath).size;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function getCacheFileName(compressionMethod: CompressionMethod): string {
|
||||||
|
return compressionMethod === CompressionMethod.Gzip
|
||||||
|
? CacheFilename.Gzip
|
||||||
|
: CacheFilename.Zstd;
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function createTempDirectory(): Promise<string> {
|
||||||
|
const IS_WINDOWS = process.platform === "win32";
|
||||||
|
|
||||||
|
let tempDirectory: string = process.env["RUNNER_TEMP"] || "";
|
||||||
|
|
||||||
|
if (!tempDirectory) {
|
||||||
|
let baseLocation: string;
|
||||||
|
if (IS_WINDOWS) {
|
||||||
|
// On Windows use the USERPROFILE env variable
|
||||||
|
baseLocation = process.env["USERPROFILE"] || "C:\\";
|
||||||
|
} else {
|
||||||
|
if (process.platform === "darwin") {
|
||||||
|
baseLocation = "/Users";
|
||||||
|
} else {
|
||||||
|
baseLocation = "/home";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
tempDirectory = path.join(baseLocation, "actions", "temp");
|
||||||
|
}
|
||||||
|
|
||||||
|
const dest = path.join(tempDirectory, uuidV4());
|
||||||
|
await io.mkdirP(dest);
|
||||||
|
return dest;
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function resolvePaths(patterns: string[]): Promise<string[]> {
|
||||||
|
const paths: string[] = [];
|
||||||
|
const workspace = process.env["GITHUB_WORKSPACE"] ?? process.cwd();
|
||||||
|
const globber = await glob.create(patterns.join("\n"), {
|
||||||
|
implicitDescendants: false
|
||||||
|
});
|
||||||
|
|
||||||
|
for await (const file of globber.globGenerator()) {
|
||||||
|
const relativeFile = path
|
||||||
|
.relative(workspace, file)
|
||||||
|
.replace(new RegExp(`\\${path.sep}`, "g"), "/");
|
||||||
|
core.debug(`Matched: ${relativeFile}`);
|
||||||
|
// Paths are made relative so the tar entries are all relative to the root of the workspace.
|
||||||
|
paths.push(`${relativeFile}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
return paths;
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function isZstdInstalled(): Promise<boolean> {
|
||||||
|
try {
|
||||||
|
await io.which("zstd", true);
|
||||||
|
return true;
|
||||||
|
} catch (error) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function isGnuTarInstalled(): Promise<boolean> {
|
||||||
|
const versionOutput = await getVersion("tar");
|
||||||
|
return versionOutput.toLowerCase().includes("gnu tar");
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function getCompressionMethod(): Promise<CompressionMethod> {
|
||||||
|
if (
|
||||||
|
(process.platform === "win32" && !(await isGnuTarInstalled())) ||
|
||||||
|
!(await isZstdInstalled())
|
||||||
|
) {
|
||||||
|
// Disable zstd due to bug https://github.com/actions/cache/issues/301
|
||||||
|
return CompressionMethod.Gzip;
|
||||||
|
}
|
||||||
|
|
||||||
|
const versionOutput = await getVersion("zstd");
|
||||||
|
const version = semver.clean(versionOutput);
|
||||||
|
|
||||||
|
// zstd is installed but using a version earlier than v1.3.2
|
||||||
|
// v1.3.2 is required to use the `--long` options in zstd
|
||||||
|
return !version || semver.lt(version, "v1.3.2")
|
||||||
|
? CompressionMethod.ZstdWithoutLong
|
||||||
|
: CompressionMethod.Zstd;
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function unlinkFile(filePath: fs.PathLike): Promise<void> {
|
||||||
|
return await fs.promises.unlink(filePath);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function assertDefined<T>(name: string, value?: T): T {
|
||||||
|
if (value === undefined) {
|
||||||
|
throw Error(`Expected ${name} but value was undefiend`);
|
||||||
|
}
|
||||||
|
|
||||||
|
return value;
|
||||||
|
}
|
||||||
@ -0,0 +1,557 @@
|
|||||||
|
import * as core from "@actions/core";
|
||||||
|
import { HttpClient } from "@actions/http-client";
|
||||||
|
import { BearerCredentialHandler } from "@actions/http-client/lib/auth";
|
||||||
|
import {
|
||||||
|
RequestOptions,
|
||||||
|
TypedResponse
|
||||||
|
} from "@actions/http-client/lib/interfaces";
|
||||||
|
import {
|
||||||
|
_Object,
|
||||||
|
ListObjectsV2Command,
|
||||||
|
ListObjectsV2CommandInput,
|
||||||
|
ListObjectsV2CommandOutput,
|
||||||
|
S3Client,
|
||||||
|
S3ClientConfig
|
||||||
|
} from "@aws-sdk/client-s3";
|
||||||
|
import { Progress, Upload } from "@aws-sdk/lib-storage";
|
||||||
|
import * as crypto from "crypto";
|
||||||
|
import * as fs from "fs";
|
||||||
|
import { URL } from "url";
|
||||||
|
|
||||||
|
import * as utils from "./cacheUtils";
|
||||||
|
import { CompressionMethod } from "./cacheUtils";
|
||||||
|
import {
|
||||||
|
ArtifactCacheEntry,
|
||||||
|
CommitCacheRequest,
|
||||||
|
DownloadOptions,
|
||||||
|
InternalCacheOptions,
|
||||||
|
ITypedResponseWithError,
|
||||||
|
ReserveCacheRequest,
|
||||||
|
ReserveCacheResponse,
|
||||||
|
UploadOptions
|
||||||
|
} from "./contracts";
|
||||||
|
import {
|
||||||
|
downloadCacheHttpClient,
|
||||||
|
downloadCacheStorageS3
|
||||||
|
} from "./downloadUtils";
|
||||||
|
import { getDownloadOptions, getUploadOptions } from "./options";
|
||||||
|
import {
|
||||||
|
isSuccessStatusCode,
|
||||||
|
retryHttpClientResponse,
|
||||||
|
retryTypedResponse
|
||||||
|
} from "./requestUtils";
|
||||||
|
|
||||||
|
const versionSalt = "1.0";
|
||||||
|
|
||||||
|
function getCacheApiUrl(resource: string): string {
|
||||||
|
const baseUrl: string = process.env["ACTIONS_CACHE_URL"] || "";
|
||||||
|
if (!baseUrl) {
|
||||||
|
throw new Error(
|
||||||
|
"Cache Service Url not found, unable to restore cache."
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
const url = `${baseUrl}_apis/artifactcache/${resource}`;
|
||||||
|
core.debug(`Resource Url: ${url}`);
|
||||||
|
return url;
|
||||||
|
}
|
||||||
|
|
||||||
|
function createAcceptHeader(type: string, apiVersion: string): string {
|
||||||
|
return `${type};api-version=${apiVersion}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
function getRequestOptions(): RequestOptions {
|
||||||
|
const requestOptions: RequestOptions = {
|
||||||
|
headers: {
|
||||||
|
Accept: createAcceptHeader("application/json", "6.0-preview.1")
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
return requestOptions;
|
||||||
|
}
|
||||||
|
|
||||||
|
function createHttpClient(): HttpClient {
|
||||||
|
const token = process.env["ACTIONS_RUNTIME_TOKEN"] || "";
|
||||||
|
const bearerCredentialHandler = new BearerCredentialHandler(token);
|
||||||
|
|
||||||
|
return new HttpClient(
|
||||||
|
"actions/cache",
|
||||||
|
[bearerCredentialHandler],
|
||||||
|
getRequestOptions()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function getCacheVersion(
|
||||||
|
paths: string[],
|
||||||
|
compressionMethod?: CompressionMethod
|
||||||
|
): string {
|
||||||
|
const components = paths.concat(
|
||||||
|
!compressionMethod || compressionMethod === CompressionMethod.Gzip
|
||||||
|
? []
|
||||||
|
: [compressionMethod]
|
||||||
|
);
|
||||||
|
|
||||||
|
// Add salt to cache version to support breaking changes in cache entry
|
||||||
|
components.push(versionSalt);
|
||||||
|
|
||||||
|
return crypto
|
||||||
|
.createHash("sha256")
|
||||||
|
.update(components.join("|"))
|
||||||
|
.digest("hex");
|
||||||
|
}
|
||||||
|
|
||||||
|
interface _content {
|
||||||
|
Key?: string;
|
||||||
|
LastModified?: Date;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function getCacheEntryS3(
|
||||||
|
s3Options: S3ClientConfig,
|
||||||
|
s3BucketName: string,
|
||||||
|
keys: string[],
|
||||||
|
paths: string[]
|
||||||
|
): Promise<ArtifactCacheEntry | null> {
|
||||||
|
const primaryKey = keys[0];
|
||||||
|
|
||||||
|
const s3client = new S3Client(s3Options);
|
||||||
|
|
||||||
|
const contents: _content[] = [];
|
||||||
|
let s3ContinuationToken: string | undefined | null = null;
|
||||||
|
let count = 0;
|
||||||
|
|
||||||
|
const param = {
|
||||||
|
Bucket: s3BucketName
|
||||||
|
} as ListObjectsV2CommandInput;
|
||||||
|
|
||||||
|
for (;;) {
|
||||||
|
core.debug(`ListObjects Count: ${count}`);
|
||||||
|
if (s3ContinuationToken != null) {
|
||||||
|
param.ContinuationToken = s3ContinuationToken;
|
||||||
|
}
|
||||||
|
|
||||||
|
let response: ListObjectsV2CommandOutput;
|
||||||
|
try {
|
||||||
|
response = await s3client.send(new ListObjectsV2Command(param));
|
||||||
|
} catch (e) {
|
||||||
|
throw new Error(`Error from S3: ${e}`);
|
||||||
|
}
|
||||||
|
if (!response.Contents) {
|
||||||
|
return null;
|
||||||
|
// throw new Error(`Cannot found object in bucket ${s3BucketName}`);
|
||||||
|
}
|
||||||
|
core.debug(`Found objects ${response.Contents.length}`);
|
||||||
|
|
||||||
|
const found = response.Contents.find(
|
||||||
|
(content: _Object) => content.Key === primaryKey
|
||||||
|
);
|
||||||
|
if (found && found.LastModified) {
|
||||||
|
return {
|
||||||
|
cacheKey: primaryKey,
|
||||||
|
creationTime: found.LastModified.toString()
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
response.Contents.map((obj: _Object) =>
|
||||||
|
contents.push({
|
||||||
|
Key: obj.Key,
|
||||||
|
LastModified: obj.LastModified
|
||||||
|
})
|
||||||
|
);
|
||||||
|
core.debug(`Total objects ${contents.length}`);
|
||||||
|
|
||||||
|
if (response.IsTruncated) {
|
||||||
|
s3ContinuationToken = response.NextContinuationToken;
|
||||||
|
} else {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
count++;
|
||||||
|
}
|
||||||
|
|
||||||
|
core.debug("Not found in primary key, will fallback to restore keys");
|
||||||
|
const notPrimaryKey = keys.slice(1);
|
||||||
|
const found = searchRestoreKeyEntry(notPrimaryKey, contents);
|
||||||
|
if (found != null && found.LastModified) {
|
||||||
|
return {
|
||||||
|
cacheKey: found.Key,
|
||||||
|
creationTime: found.LastModified.toString()
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
function searchRestoreKeyEntry(
|
||||||
|
notPrimaryKey: string[],
|
||||||
|
entries: _content[]
|
||||||
|
): _content | null {
|
||||||
|
for (const k of notPrimaryKey) {
|
||||||
|
const found = _searchRestoreKeyEntry(k, entries);
|
||||||
|
if (found != null) {
|
||||||
|
return found;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
function _searchRestoreKeyEntry(
|
||||||
|
notPrimaryKey: string,
|
||||||
|
entries: _content[]
|
||||||
|
): _content | null {
|
||||||
|
const matchPrefix: _content[] = [];
|
||||||
|
|
||||||
|
for (const entry of entries) {
|
||||||
|
if (entry.Key === notPrimaryKey) {
|
||||||
|
// extractly match, Use this entry
|
||||||
|
return entry;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (entry.Key?.startsWith(notPrimaryKey)) {
|
||||||
|
matchPrefix.push(entry);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (matchPrefix.length === 0) {
|
||||||
|
// not found, go to next key
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
matchPrefix.sort(function (i, j) {
|
||||||
|
if (i.LastModified == undefined || j.LastModified == undefined) {
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
if (i.LastModified?.getTime() === j.LastModified?.getTime()) {
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
if (i.LastModified?.getTime() > j.LastModified?.getTime()) {
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
if (i.LastModified?.getTime() < j.LastModified?.getTime()) {
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
return 0;
|
||||||
|
});
|
||||||
|
|
||||||
|
// return newest entry
|
||||||
|
return matchPrefix[0];
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function getCacheEntry(
|
||||||
|
keys: string[],
|
||||||
|
paths: string[],
|
||||||
|
options?: InternalCacheOptions,
|
||||||
|
s3Options?: S3ClientConfig,
|
||||||
|
s3BucketName?: string
|
||||||
|
): Promise<ArtifactCacheEntry | null> {
|
||||||
|
if (s3Options && s3BucketName) {
|
||||||
|
return await getCacheEntryS3(s3Options, s3BucketName, keys, paths);
|
||||||
|
}
|
||||||
|
|
||||||
|
const httpClient = createHttpClient();
|
||||||
|
const version = getCacheVersion(paths, options?.compressionMethod);
|
||||||
|
const resource = `cache?keys=${encodeURIComponent(
|
||||||
|
keys.join(",")
|
||||||
|
)}&version=${version}`;
|
||||||
|
|
||||||
|
const response = await retryTypedResponse("getCacheEntry", async () =>
|
||||||
|
httpClient.getJson<ArtifactCacheEntry>(getCacheApiUrl(resource))
|
||||||
|
);
|
||||||
|
if (response.statusCode === 204) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
if (!isSuccessStatusCode(response.statusCode)) {
|
||||||
|
throw new Error(`Cache service responded with ${response.statusCode}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
const cacheResult = response.result;
|
||||||
|
const cacheDownloadUrl = cacheResult?.archiveLocation;
|
||||||
|
if (!cacheDownloadUrl) {
|
||||||
|
throw new Error("Cache not found.");
|
||||||
|
}
|
||||||
|
core.setSecret(cacheDownloadUrl);
|
||||||
|
core.debug(`Cache Result:`);
|
||||||
|
core.debug(JSON.stringify(cacheResult));
|
||||||
|
|
||||||
|
return cacheResult;
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function downloadCache(
|
||||||
|
cacheEntry: ArtifactCacheEntry,
|
||||||
|
archivePath: string,
|
||||||
|
options?: DownloadOptions,
|
||||||
|
s3Options?: S3ClientConfig,
|
||||||
|
s3BucketName?: string
|
||||||
|
): Promise<void> {
|
||||||
|
const archiveLocation = cacheEntry.archiveLocation ?? "https://example.com"; // for dummy
|
||||||
|
const archiveUrl = new URL(archiveLocation);
|
||||||
|
const downloadOptions = getDownloadOptions(options);
|
||||||
|
|
||||||
|
if (s3Options && s3BucketName && cacheEntry.cacheKey) {
|
||||||
|
await downloadCacheStorageS3(
|
||||||
|
cacheEntry.cacheKey,
|
||||||
|
archivePath,
|
||||||
|
s3Options,
|
||||||
|
s3BucketName
|
||||||
|
);
|
||||||
|
} else {
|
||||||
|
// Otherwise, download using the Actions http-client.
|
||||||
|
await downloadCacheHttpClient(archiveLocation, archivePath);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Reserve Cache
|
||||||
|
export async function reserveCache(
|
||||||
|
key: string,
|
||||||
|
paths: string[],
|
||||||
|
options?: InternalCacheOptions,
|
||||||
|
s3Options?: S3ClientConfig,
|
||||||
|
s3BucketName?: string
|
||||||
|
): Promise<ITypedResponseWithError<ReserveCacheResponse>> {
|
||||||
|
if (s3Options && s3BucketName) {
|
||||||
|
return {
|
||||||
|
statusCode: 200,
|
||||||
|
result: null,
|
||||||
|
headers: {}
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
const httpClient = createHttpClient();
|
||||||
|
const version = getCacheVersion(paths, options?.compressionMethod);
|
||||||
|
|
||||||
|
const reserveCacheRequest: ReserveCacheRequest = {
|
||||||
|
key,
|
||||||
|
version,
|
||||||
|
cacheSize: options?.cacheSize
|
||||||
|
};
|
||||||
|
const response = await retryTypedResponse("reserveCache", async () =>
|
||||||
|
httpClient.postJson<ReserveCacheResponse>(
|
||||||
|
getCacheApiUrl("caches"),
|
||||||
|
reserveCacheRequest
|
||||||
|
)
|
||||||
|
);
|
||||||
|
return response;
|
||||||
|
}
|
||||||
|
|
||||||
|
function getContentRange(start: number, end: number): string {
|
||||||
|
// Format: `bytes start-end/filesize
|
||||||
|
// start and end are inclusive
|
||||||
|
// filesize can be *
|
||||||
|
// For a 200 byte chunk starting at byte 0:
|
||||||
|
// Content-Range: bytes 0-199/*
|
||||||
|
return `bytes ${start}-${end}/*`;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function uploadChunk(
|
||||||
|
httpClient: HttpClient,
|
||||||
|
resourceUrl: string,
|
||||||
|
openStream: () => NodeJS.ReadableStream,
|
||||||
|
start: number,
|
||||||
|
end: number
|
||||||
|
): Promise<void> {
|
||||||
|
core.debug(
|
||||||
|
`Uploading chunk of size ${
|
||||||
|
end - start + 1
|
||||||
|
} bytes at offset ${start} with content range: ${getContentRange(
|
||||||
|
start,
|
||||||
|
end
|
||||||
|
)}`
|
||||||
|
);
|
||||||
|
const additionalHeaders = {
|
||||||
|
"Content-Type": "application/octet-stream",
|
||||||
|
"Content-Range": getContentRange(start, end)
|
||||||
|
};
|
||||||
|
|
||||||
|
const uploadChunkResponse = await retryHttpClientResponse(
|
||||||
|
`uploadChunk (start: ${start}, end: ${end})`,
|
||||||
|
async () =>
|
||||||
|
httpClient.sendStream(
|
||||||
|
"PATCH",
|
||||||
|
resourceUrl,
|
||||||
|
openStream(),
|
||||||
|
additionalHeaders
|
||||||
|
)
|
||||||
|
);
|
||||||
|
|
||||||
|
if (!isSuccessStatusCode(uploadChunkResponse.message.statusCode)) {
|
||||||
|
throw new Error(
|
||||||
|
`Cache service responded with ${uploadChunkResponse.message.statusCode} during upload chunk.`
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function uploadFileS3(
|
||||||
|
s3options: S3ClientConfig,
|
||||||
|
s3BucketName: string,
|
||||||
|
archivePath: string,
|
||||||
|
key: string,
|
||||||
|
concurrency: number,
|
||||||
|
maxChunkSize: number
|
||||||
|
): Promise<void> {
|
||||||
|
core.debug(`Start upload to S3 (bucket: ${s3BucketName})`);
|
||||||
|
|
||||||
|
const fileStream = fs.createReadStream(archivePath);
|
||||||
|
|
||||||
|
try {
|
||||||
|
const parallelUpload = new Upload({
|
||||||
|
client: new S3Client(s3options),
|
||||||
|
queueSize: concurrency,
|
||||||
|
partSize: maxChunkSize,
|
||||||
|
|
||||||
|
params: {
|
||||||
|
Bucket: s3BucketName,
|
||||||
|
Key: key,
|
||||||
|
Body: fileStream
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
parallelUpload.on("httpUploadProgress", (progress: Progress) => {
|
||||||
|
core.debug(`Uploading chunk progress: ${JSON.stringify(progress)}`);
|
||||||
|
});
|
||||||
|
|
||||||
|
await parallelUpload.done();
|
||||||
|
} catch (error) {
|
||||||
|
throw new Error(`Cache upload failed because ${error}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function uploadFile(
|
||||||
|
httpClient: HttpClient,
|
||||||
|
cacheId: number,
|
||||||
|
archivePath: string,
|
||||||
|
key: string,
|
||||||
|
options?: UploadOptions,
|
||||||
|
s3options?: S3ClientConfig,
|
||||||
|
s3BucketName?: string
|
||||||
|
): Promise<void> {
|
||||||
|
// Upload Chunks
|
||||||
|
const uploadOptions = getUploadOptions(options);
|
||||||
|
|
||||||
|
const concurrency = utils.assertDefined(
|
||||||
|
"uploadConcurrency",
|
||||||
|
uploadOptions.uploadConcurrency
|
||||||
|
);
|
||||||
|
const maxChunkSize = utils.assertDefined(
|
||||||
|
"uploadChunkSize",
|
||||||
|
uploadOptions.uploadChunkSize
|
||||||
|
);
|
||||||
|
|
||||||
|
const parallelUploads = [...new Array(concurrency).keys()];
|
||||||
|
core.debug("Awaiting all uploads");
|
||||||
|
let offset = 0;
|
||||||
|
|
||||||
|
if (s3options && s3BucketName) {
|
||||||
|
await uploadFileS3(
|
||||||
|
s3options,
|
||||||
|
s3BucketName,
|
||||||
|
archivePath,
|
||||||
|
key,
|
||||||
|
concurrency,
|
||||||
|
maxChunkSize
|
||||||
|
);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const fileSize = utils.getArchiveFileSizeInBytes(archivePath);
|
||||||
|
const resourceUrl = getCacheApiUrl(`caches/${cacheId.toString()}`);
|
||||||
|
const fd = fs.openSync(archivePath, "r");
|
||||||
|
try {
|
||||||
|
await Promise.all(
|
||||||
|
parallelUploads.map(async () => {
|
||||||
|
while (offset < fileSize) {
|
||||||
|
const chunkSize = Math.min(fileSize - offset, maxChunkSize);
|
||||||
|
const start = offset;
|
||||||
|
const end = offset + chunkSize - 1;
|
||||||
|
offset += maxChunkSize;
|
||||||
|
|
||||||
|
await uploadChunk(
|
||||||
|
httpClient,
|
||||||
|
resourceUrl,
|
||||||
|
() =>
|
||||||
|
fs
|
||||||
|
.createReadStream(archivePath, {
|
||||||
|
fd,
|
||||||
|
start,
|
||||||
|
end,
|
||||||
|
autoClose: false
|
||||||
|
})
|
||||||
|
.on("error", error => {
|
||||||
|
throw new Error(
|
||||||
|
`Cache upload failed because file read failed with ${error.message}`
|
||||||
|
);
|
||||||
|
}),
|
||||||
|
start,
|
||||||
|
end
|
||||||
|
);
|
||||||
|
}
|
||||||
|
})
|
||||||
|
);
|
||||||
|
} finally {
|
||||||
|
fs.closeSync(fd);
|
||||||
|
}
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function commitCache(
|
||||||
|
httpClient: HttpClient,
|
||||||
|
cacheId: number,
|
||||||
|
filesize: number
|
||||||
|
): Promise<TypedResponse<null>> {
|
||||||
|
const commitCacheRequest: CommitCacheRequest = { size: filesize };
|
||||||
|
return await retryTypedResponse("commitCache", async () =>
|
||||||
|
httpClient.postJson<null>(
|
||||||
|
getCacheApiUrl(`caches/${cacheId.toString()}`),
|
||||||
|
commitCacheRequest
|
||||||
|
)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function saveCache(
|
||||||
|
cacheId: number,
|
||||||
|
archivePath: string,
|
||||||
|
key: string,
|
||||||
|
options?: UploadOptions,
|
||||||
|
s3Options?: S3ClientConfig,
|
||||||
|
s3BucketName?: string
|
||||||
|
): Promise<void> {
|
||||||
|
const httpClient = createHttpClient();
|
||||||
|
|
||||||
|
core.debug("Upload cache");
|
||||||
|
await uploadFile(
|
||||||
|
httpClient,
|
||||||
|
cacheId,
|
||||||
|
archivePath,
|
||||||
|
key,
|
||||||
|
options,
|
||||||
|
s3Options,
|
||||||
|
s3BucketName
|
||||||
|
);
|
||||||
|
|
||||||
|
// Commit Cache
|
||||||
|
core.debug("Commiting cache");
|
||||||
|
const cacheSize = utils.getArchiveFileSizeInBytes(archivePath);
|
||||||
|
core.info(
|
||||||
|
`Cache Size: ~${Math.round(
|
||||||
|
cacheSize / (1024 * 1024)
|
||||||
|
)} MB (${cacheSize} B)`
|
||||||
|
);
|
||||||
|
|
||||||
|
if (!s3Options) {
|
||||||
|
// already commit on S3
|
||||||
|
const commitCacheResponse = await commitCache(
|
||||||
|
httpClient,
|
||||||
|
cacheId,
|
||||||
|
cacheSize
|
||||||
|
);
|
||||||
|
if (!isSuccessStatusCode(commitCacheResponse.statusCode)) {
|
||||||
|
throw new Error(
|
||||||
|
`Cache service responded with ${commitCacheResponse.statusCode} during commit cache.`
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
core.info("Cache saved successfully");
|
||||||
|
}
|
||||||
@ -0,0 +1,84 @@
|
|||||||
|
import { HttpClientError } from "@actions/http-client";
|
||||||
|
import { TypedResponse } from "@actions/http-client/lib/interfaces";
|
||||||
|
|
||||||
|
import { CompressionMethod } from "./cacheUtils";
|
||||||
|
|
||||||
|
export interface ITypedResponseWithError<T> extends TypedResponse<T> {
|
||||||
|
error?: HttpClientError;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface ArtifactCacheEntry {
|
||||||
|
cacheKey?: string;
|
||||||
|
scope?: string;
|
||||||
|
creationTime?: string;
|
||||||
|
archiveLocation?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface CommitCacheRequest {
|
||||||
|
size: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface ReserveCacheRequest {
|
||||||
|
key: string;
|
||||||
|
version?: string;
|
||||||
|
cacheSize?: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface ReserveCacheResponse {
|
||||||
|
cacheId: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface InternalCacheOptions {
|
||||||
|
compressionMethod?: CompressionMethod;
|
||||||
|
cacheSize?: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Options to control cache upload
|
||||||
|
*/
|
||||||
|
export interface UploadOptions {
|
||||||
|
/**
|
||||||
|
* Number of parallel cache upload
|
||||||
|
*
|
||||||
|
* @default 4
|
||||||
|
*/
|
||||||
|
uploadConcurrency?: number;
|
||||||
|
/**
|
||||||
|
* Maximum chunk size in bytes for cache upload
|
||||||
|
*
|
||||||
|
* @default 32MB
|
||||||
|
*/
|
||||||
|
uploadChunkSize?: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Options to control cache download
|
||||||
|
*/
|
||||||
|
export interface DownloadOptions {
|
||||||
|
/**
|
||||||
|
* Indicates whether to use the Azure Blob SDK to download caches
|
||||||
|
* that are stored on Azure Blob Storage to improve reliability and
|
||||||
|
* performance
|
||||||
|
*
|
||||||
|
* @default true
|
||||||
|
*/
|
||||||
|
useAzureSdk?: boolean;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Number of parallel downloads (this option only applies when using
|
||||||
|
* the Azure SDK)
|
||||||
|
*
|
||||||
|
* @default 8
|
||||||
|
*/
|
||||||
|
downloadConcurrency?: number;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Maximum time for each download request, in milliseconds (this
|
||||||
|
* option only applies when using the Azure SDK)
|
||||||
|
*
|
||||||
|
* @default 30000
|
||||||
|
*/
|
||||||
|
timeoutInMs?: number;
|
||||||
|
|
||||||
|
lookupOnly?: boolean;
|
||||||
|
}
|
||||||
@ -0,0 +1,104 @@
|
|||||||
|
import * as core from "@actions/core";
|
||||||
|
import { HttpClient, HttpClientResponse } from "@actions/http-client";
|
||||||
|
import { GetObjectCommand, S3Client, S3ClientConfig } from "@aws-sdk/client-s3";
|
||||||
|
import * as fs from "fs";
|
||||||
|
import * as stream from "stream";
|
||||||
|
import * as util from "util";
|
||||||
|
|
||||||
|
import * as utils from "./cacheUtils";
|
||||||
|
import { SocketTimeout } from "./cacheUtils";
|
||||||
|
import { retryHttpClientResponse } from "./requestUtils";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Pipes the body of a HTTP response to a stream
|
||||||
|
*
|
||||||
|
* @param response the HTTP response
|
||||||
|
* @param output the writable stream
|
||||||
|
*/
|
||||||
|
async function pipeResponseToStream(
|
||||||
|
response: HttpClientResponse,
|
||||||
|
output: NodeJS.WritableStream
|
||||||
|
): Promise<void> {
|
||||||
|
const pipeline = util.promisify(stream.pipeline);
|
||||||
|
await pipeline(response.message, output);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Download the cache using the Actions toolkit http-client
|
||||||
|
*
|
||||||
|
* @param archiveLocation the URL for the cache
|
||||||
|
* @param archivePath the local path where the cache is saved
|
||||||
|
*/
|
||||||
|
export async function downloadCacheHttpClient(
|
||||||
|
archiveLocation: string,
|
||||||
|
archivePath: string
|
||||||
|
): Promise<void> {
|
||||||
|
const writeStream = fs.createWriteStream(archivePath);
|
||||||
|
const httpClient = new HttpClient("actions/cache");
|
||||||
|
const downloadResponse = await retryHttpClientResponse(
|
||||||
|
"downloadCache",
|
||||||
|
async () => httpClient.get(archiveLocation)
|
||||||
|
);
|
||||||
|
|
||||||
|
// Abort download if no traffic received over the socket.
|
||||||
|
downloadResponse.message.socket.setTimeout(SocketTimeout, () => {
|
||||||
|
downloadResponse.message.destroy();
|
||||||
|
core.debug(
|
||||||
|
`Aborting download, socket timed out after ${SocketTimeout} ms`
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
await pipeResponseToStream(downloadResponse, writeStream);
|
||||||
|
|
||||||
|
// Validate download size.
|
||||||
|
const contentLengthHeader =
|
||||||
|
downloadResponse.message.headers["content-length"];
|
||||||
|
|
||||||
|
if (contentLengthHeader) {
|
||||||
|
const expectedLength = parseInt(contentLengthHeader);
|
||||||
|
const actualLength = utils.getArchiveFileSizeInBytes(archivePath);
|
||||||
|
|
||||||
|
if (actualLength !== expectedLength) {
|
||||||
|
throw new Error(
|
||||||
|
`Incomplete download. Expected file size: ${expectedLength}, actual file size: ${actualLength}`
|
||||||
|
);
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
core.debug("Unable to validate download, no Content-Length header");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Download the cache using the AWS S3. Only call this method if the use S3.
|
||||||
|
*
|
||||||
|
* @param key the key for the cache in S3
|
||||||
|
* @param archivePath the local path where the cache is saved
|
||||||
|
* @param s3Options: the option for AWS S3 client
|
||||||
|
* @param s3BucketName: the name of bucket in AWS S3
|
||||||
|
*/
|
||||||
|
export async function downloadCacheStorageS3(
|
||||||
|
key: string,
|
||||||
|
archivePath: string,
|
||||||
|
s3Options: S3ClientConfig,
|
||||||
|
s3BucketName: string
|
||||||
|
): Promise<void> {
|
||||||
|
const s3client = new S3Client(s3Options);
|
||||||
|
const param = {
|
||||||
|
Bucket: s3BucketName,
|
||||||
|
Key: key
|
||||||
|
};
|
||||||
|
|
||||||
|
const response = await s3client.send(new GetObjectCommand(param));
|
||||||
|
if (!response.Body) {
|
||||||
|
throw new Error(
|
||||||
|
`Incomplete download. response.Body is undefined from S3.`
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
const fileStream = fs.createWriteStream(archivePath);
|
||||||
|
|
||||||
|
const pipeline = util.promisify(stream.pipeline);
|
||||||
|
await pipeline(response.Body as stream.Readable, fileStream);
|
||||||
|
|
||||||
|
return;
|
||||||
|
}
|
||||||
@ -0,0 +1,63 @@
|
|||||||
|
import * as core from "@actions/core";
|
||||||
|
|
||||||
|
import { DownloadOptions, UploadOptions } from "./contracts";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Returns a copy of the upload options with defaults filled in.
|
||||||
|
*
|
||||||
|
* @param copy the original upload options
|
||||||
|
*/
|
||||||
|
export function getUploadOptions(copy?: UploadOptions): UploadOptions {
|
||||||
|
const result: UploadOptions = {
|
||||||
|
uploadConcurrency: 4,
|
||||||
|
uploadChunkSize: 32 * 1024 * 1024
|
||||||
|
};
|
||||||
|
|
||||||
|
if (copy) {
|
||||||
|
if (typeof copy.uploadConcurrency === "number") {
|
||||||
|
result.uploadConcurrency = copy.uploadConcurrency;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (typeof copy.uploadChunkSize === "number") {
|
||||||
|
result.uploadChunkSize = copy.uploadChunkSize;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
core.debug(`Upload concurrency: ${result.uploadConcurrency}`);
|
||||||
|
core.debug(`Upload chunk size: ${result.uploadChunkSize}`);
|
||||||
|
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Returns a copy of the download options with defaults filled in.
|
||||||
|
*
|
||||||
|
* @param copy the original download options
|
||||||
|
*/
|
||||||
|
export function getDownloadOptions(copy?: DownloadOptions): DownloadOptions {
|
||||||
|
const result: DownloadOptions = {
|
||||||
|
useAzureSdk: true,
|
||||||
|
downloadConcurrency: 8,
|
||||||
|
timeoutInMs: 30000
|
||||||
|
};
|
||||||
|
|
||||||
|
if (copy) {
|
||||||
|
if (typeof copy.useAzureSdk === "boolean") {
|
||||||
|
result.useAzureSdk = copy.useAzureSdk;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (typeof copy.downloadConcurrency === "number") {
|
||||||
|
result.downloadConcurrency = copy.downloadConcurrency;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (typeof copy.timeoutInMs === "number") {
|
||||||
|
result.timeoutInMs = copy.timeoutInMs;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
core.debug(`Use Azure SDK: ${result.useAzureSdk}`);
|
||||||
|
core.debug(`Download concurrency: ${result.downloadConcurrency}`);
|
||||||
|
core.debug(`Request timeout (ms): ${result.timeoutInMs}`);
|
||||||
|
|
||||||
|
return result;
|
||||||
|
}
|
||||||
@ -0,0 +1,139 @@
|
|||||||
|
import * as core from "@actions/core";
|
||||||
|
import {
|
||||||
|
HttpClientError,
|
||||||
|
HttpClientResponse,
|
||||||
|
HttpCodes
|
||||||
|
} from "@actions/http-client";
|
||||||
|
|
||||||
|
import { DefaultRetryAttempts, DefaultRetryDelay } from "./cacheUtils";
|
||||||
|
import { ITypedResponseWithError } from "./contracts";
|
||||||
|
|
||||||
|
export function isSuccessStatusCode(statusCode?: number): boolean {
|
||||||
|
if (!statusCode) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
return statusCode >= 200 && statusCode < 300;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function isServerErrorStatusCode(statusCode?: number): boolean {
|
||||||
|
if (!statusCode) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
return statusCode >= 500;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function isRetryableStatusCode(statusCode?: number): boolean {
|
||||||
|
if (!statusCode) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
const retryableStatusCodes = [
|
||||||
|
HttpCodes.BadGateway,
|
||||||
|
HttpCodes.ServiceUnavailable,
|
||||||
|
HttpCodes.GatewayTimeout
|
||||||
|
];
|
||||||
|
return retryableStatusCodes.includes(statusCode);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function sleep(milliseconds: number): Promise<void> {
|
||||||
|
return new Promise(resolve => setTimeout(resolve, milliseconds));
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function retry<T>(
|
||||||
|
name: string,
|
||||||
|
method: () => Promise<T>,
|
||||||
|
getStatusCode: (arg0: T) => number | undefined,
|
||||||
|
maxAttempts = DefaultRetryAttempts,
|
||||||
|
delay = DefaultRetryDelay,
|
||||||
|
onError: ((arg0: Error) => T | undefined) | undefined = undefined
|
||||||
|
): Promise<T> {
|
||||||
|
let errorMessage = "";
|
||||||
|
let attempt = 1;
|
||||||
|
|
||||||
|
while (attempt <= maxAttempts) {
|
||||||
|
let response: T | undefined = undefined;
|
||||||
|
let statusCode: number | undefined = undefined;
|
||||||
|
let isRetryable = false;
|
||||||
|
|
||||||
|
try {
|
||||||
|
response = await method();
|
||||||
|
} catch (error: any) {
|
||||||
|
if (onError) {
|
||||||
|
response = onError(error);
|
||||||
|
}
|
||||||
|
|
||||||
|
isRetryable = true;
|
||||||
|
errorMessage = error.message;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (response) {
|
||||||
|
statusCode = getStatusCode(response);
|
||||||
|
|
||||||
|
if (!isServerErrorStatusCode(statusCode)) {
|
||||||
|
return response;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (statusCode) {
|
||||||
|
isRetryable = isRetryableStatusCode(statusCode);
|
||||||
|
errorMessage = `Cache service responded with ${statusCode}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
core.debug(
|
||||||
|
`${name} - Attempt ${attempt} of ${maxAttempts} failed with error: ${errorMessage}`
|
||||||
|
);
|
||||||
|
|
||||||
|
if (!isRetryable) {
|
||||||
|
core.debug(`${name} - Error is not retryable`);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
await sleep(delay);
|
||||||
|
attempt++;
|
||||||
|
}
|
||||||
|
|
||||||
|
throw Error(`${name} failed: ${errorMessage}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function retryTypedResponse<T>(
|
||||||
|
name: string,
|
||||||
|
method: () => Promise<ITypedResponseWithError<T>>,
|
||||||
|
maxAttempts = DefaultRetryAttempts,
|
||||||
|
delay = DefaultRetryDelay
|
||||||
|
): Promise<ITypedResponseWithError<T>> {
|
||||||
|
return await retry(
|
||||||
|
name,
|
||||||
|
method,
|
||||||
|
(response: ITypedResponseWithError<T>) => response.statusCode,
|
||||||
|
maxAttempts,
|
||||||
|
delay,
|
||||||
|
// If the error object contains the statusCode property, extract it and return
|
||||||
|
// an TypedResponse<T> so it can be processed by the retry logic.
|
||||||
|
(error: Error) => {
|
||||||
|
if (error instanceof HttpClientError) {
|
||||||
|
return {
|
||||||
|
statusCode: error.statusCode,
|
||||||
|
result: null,
|
||||||
|
headers: {},
|
||||||
|
error
|
||||||
|
};
|
||||||
|
} else {
|
||||||
|
return undefined;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function retryHttpClientResponse(
|
||||||
|
name: string,
|
||||||
|
method: () => Promise<HttpClientResponse>,
|
||||||
|
maxAttempts = DefaultRetryAttempts,
|
||||||
|
delay = DefaultRetryDelay
|
||||||
|
): Promise<HttpClientResponse> {
|
||||||
|
return await retry(
|
||||||
|
name,
|
||||||
|
method,
|
||||||
|
(response: HttpClientResponse) => response.message.statusCode,
|
||||||
|
maxAttempts,
|
||||||
|
delay
|
||||||
|
);
|
||||||
|
}
|
||||||
@ -0,0 +1,158 @@
|
|||||||
|
import { exec } from "@actions/exec";
|
||||||
|
import * as io from "@actions/io";
|
||||||
|
import { existsSync, writeFileSync } from "fs";
|
||||||
|
import * as path from "path";
|
||||||
|
|
||||||
|
import * as utils from "./cacheUtils";
|
||||||
|
import { CompressionMethod } from "./cacheUtils";
|
||||||
|
|
||||||
|
async function getTarPath(
|
||||||
|
args: string[],
|
||||||
|
compressionMethod: CompressionMethod
|
||||||
|
): Promise<string> {
|
||||||
|
switch (process.platform) {
|
||||||
|
case "win32": {
|
||||||
|
const systemTar = `${process.env["windir"]}\\System32\\tar.exe`;
|
||||||
|
if (compressionMethod !== CompressionMethod.Gzip) {
|
||||||
|
// We only use zstandard compression on windows when gnu tar is installed due to
|
||||||
|
// a bug with compressing large files with bsdtar + zstd
|
||||||
|
args.push("--force-local");
|
||||||
|
} else if (existsSync(systemTar)) {
|
||||||
|
return systemTar;
|
||||||
|
} else if (await utils.isGnuTarInstalled()) {
|
||||||
|
args.push("--force-local");
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
case "darwin": {
|
||||||
|
const gnuTar = await io.which("gtar", false);
|
||||||
|
if (gnuTar) {
|
||||||
|
// fix permission denied errors when extracting BSD tar archive with GNU tar - https://github.com/actions/cache/issues/527
|
||||||
|
args.push("--delay-directory-restore");
|
||||||
|
return gnuTar;
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
return await io.which("tar", true);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function execTar(
|
||||||
|
args: string[],
|
||||||
|
compressionMethod: CompressionMethod,
|
||||||
|
cwd?: string
|
||||||
|
): Promise<void> {
|
||||||
|
try {
|
||||||
|
await exec(`"${await getTarPath(args, compressionMethod)}"`, args, {
|
||||||
|
cwd
|
||||||
|
});
|
||||||
|
} catch (error: any) {
|
||||||
|
throw new Error(`Tar failed with error: ${error?.message}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function getWorkingDirectory(): string {
|
||||||
|
return process.env["GITHUB_WORKSPACE"] ?? process.cwd();
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function extractTar(
|
||||||
|
archivePath: string,
|
||||||
|
compressionMethod: CompressionMethod
|
||||||
|
): Promise<void> {
|
||||||
|
// Create directory to extract tar into
|
||||||
|
const workingDirectory = getWorkingDirectory();
|
||||||
|
await io.mkdirP(workingDirectory);
|
||||||
|
// --d: Decompress.
|
||||||
|
// --long=#: Enables long distance matching with # bits. Maximum is 30 (1GB) on 32-bit OS and 31 (2GB) on 64-bit.
|
||||||
|
// Using 30 here because we also support 32-bit self-hosted runners.
|
||||||
|
function getCompressionProgram(): string[] {
|
||||||
|
switch (compressionMethod) {
|
||||||
|
case CompressionMethod.Zstd:
|
||||||
|
return ["--use-compress-program", "zstd -d --long=30"];
|
||||||
|
case CompressionMethod.ZstdWithoutLong:
|
||||||
|
return ["--use-compress-program", "zstd -d"];
|
||||||
|
default:
|
||||||
|
return ["-z"];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
const args = [
|
||||||
|
...getCompressionProgram(),
|
||||||
|
"-xf",
|
||||||
|
archivePath.replace(new RegExp(`\\${path.sep}`, "g"), "/"),
|
||||||
|
"-P",
|
||||||
|
"-C",
|
||||||
|
workingDirectory.replace(new RegExp(`\\${path.sep}`, "g"), "/")
|
||||||
|
];
|
||||||
|
await execTar(args, compressionMethod);
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function createTar(
|
||||||
|
archiveFolder: string,
|
||||||
|
sourceDirectories: string[],
|
||||||
|
compressionMethod: CompressionMethod
|
||||||
|
): Promise<void> {
|
||||||
|
// Write source directories to manifest.txt to avoid command length limits
|
||||||
|
const manifestFilename = "manifest.txt";
|
||||||
|
const cacheFileName = utils.getCacheFileName(compressionMethod);
|
||||||
|
writeFileSync(
|
||||||
|
path.join(archiveFolder, manifestFilename),
|
||||||
|
sourceDirectories.join("\n")
|
||||||
|
);
|
||||||
|
const workingDirectory = getWorkingDirectory();
|
||||||
|
|
||||||
|
// -T#: Compress using # working thread. If # is 0, attempt to detect and use the number of physical CPU cores.
|
||||||
|
// --long=#: Enables long distance matching with # bits. Maximum is 30 (1GB) on 32-bit OS and 31 (2GB) on 64-bit.
|
||||||
|
// Using 30 here because we also support 32-bit self-hosted runners.
|
||||||
|
// Long range mode is added to zstd in v1.3.2 release, so we will not use --long in older version of zstd.
|
||||||
|
function getCompressionProgram(): string[] {
|
||||||
|
switch (compressionMethod) {
|
||||||
|
case CompressionMethod.Zstd:
|
||||||
|
return ["--use-compress-program", "zstd -T0 --long=30"];
|
||||||
|
case CompressionMethod.ZstdWithoutLong:
|
||||||
|
return ["--use-compress-program", "zstd -T0"];
|
||||||
|
default:
|
||||||
|
return ["-z"];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
const args = [
|
||||||
|
"--posix",
|
||||||
|
...getCompressionProgram(),
|
||||||
|
"-cf",
|
||||||
|
cacheFileName.replace(new RegExp(`\\${path.sep}`, "g"), "/"),
|
||||||
|
"-P",
|
||||||
|
"-C",
|
||||||
|
workingDirectory.replace(new RegExp(`\\${path.sep}`, "g"), "/"),
|
||||||
|
"--files-from",
|
||||||
|
manifestFilename
|
||||||
|
];
|
||||||
|
await execTar(args, compressionMethod, archiveFolder);
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function listTar(
|
||||||
|
archivePath: string,
|
||||||
|
compressionMethod: CompressionMethod
|
||||||
|
): Promise<void> {
|
||||||
|
// --d: Decompress.
|
||||||
|
// --long=#: Enables long distance matching with # bits.
|
||||||
|
// Maximum is 30 (1GB) on 32-bit OS and 31 (2GB) on 64-bit.
|
||||||
|
// Using 30 here because we also support 32-bit self-hosted runners.
|
||||||
|
function getCompressionProgram(): string[] {
|
||||||
|
switch (compressionMethod) {
|
||||||
|
case CompressionMethod.Zstd:
|
||||||
|
return ["--use-compress-program", "zstd -d --long=30"];
|
||||||
|
case CompressionMethod.ZstdWithoutLong:
|
||||||
|
return ["--use-compress-program", "zstd -d"];
|
||||||
|
default:
|
||||||
|
return ["-z"];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
const args = [
|
||||||
|
...getCompressionProgram(),
|
||||||
|
"-tf",
|
||||||
|
archivePath.replace(new RegExp(`\\${path.sep}`, "g"), "/"),
|
||||||
|
"-P"
|
||||||
|
];
|
||||||
|
await execTar(args, compressionMethod);
|
||||||
|
}
|
||||||
Loading…
Reference in New Issue