actions
/
cache
mirror of https://github.com/actions/cache.git
3
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Format and update tests

Browse Source
joshmgross/chunked-upload-testing
Josh Gross 5 years ago
parent 2ce22df8c4
commit cdec5dec0d
4 changed files with 108 additions and 67 deletions
Show Stats Download Patch File Download Diff File

68
__tests__/save.test.ts
Unescape Escape View File

@ -212,14 +212,14 @@ test("save with large cache outputs warning", async () => {
const IS_WINDOWS = process.platform === "win32"; const IS_WINDOWS = process.platform === "win32";
const args = IS_WINDOWS const args = IS_WINDOWS
? [ ? [
"-cz", "-cz",
"--force-local", "--force-local",
"-f", "-f",
archivePath.replace(/\\/g, "/"), archivePath.replace(/\\/g, "/"),
"-C", "-C",
cachePath.replace(/\\/g, "/"), cachePath.replace(/\\/g, "/"),
"." "."
] ]
: ["-cz", "-f", archivePath, "-C", cachePath, "."]; : ["-cz", "-f", archivePath, "-C", cachePath, "."];
expect(execMock).toHaveBeenCalledTimes(1); expect(execMock).toHaveBeenCalledTimes(1);
@ -259,6 +259,11 @@ test("save with server error outputs warning", async () => {
const cachePath = path.resolve(inputPath); const cachePath = path.resolve(inputPath);
testUtils.setInput(Inputs.Path, inputPath); testUtils.setInput(Inputs.Path, inputPath);
const cacheId = 4;
const reserveCacheMock = jest.spyOn(cacheHttpClient, "reserveCache").mockImplementationOnce(() => {
return Promise.resolve(cacheId);
});
const execMock = jest.spyOn(exec, "exec"); const execMock = jest.spyOn(exec, "exec");
const saveCacheMock = jest const saveCacheMock = jest
@ -269,26 +274,29 @@ test("save with server error outputs warning", async () => {
await run(); await run();
expect(reserveCacheMock).toHaveBeenCalledTimes(1);
expect(reserveCacheMock).toHaveBeenCalledWith(primaryKey);
const archivePath = path.join("/foo/bar", "cache.tgz"); const archivePath = path.join("/foo/bar", "cache.tgz");
const IS_WINDOWS = process.platform === "win32"; const IS_WINDOWS = process.platform === "win32";
const args = IS_WINDOWS const args = IS_WINDOWS
? [ ? [
"-cz", "-cz",
"--force-local", "--force-local",
"-f", "-f",
archivePath.replace(/\\/g, "/"), archivePath.replace(/\\/g, "/"),
"-C", "-C",
cachePath.replace(/\\/g, "/"), cachePath.replace(/\\/g, "/"),
"." "."
] ]
: ["-cz", "-f", archivePath, "-C", cachePath, "."]; : ["-cz", "-f", archivePath, "-C", cachePath, "."];
expect(execMock).toHaveBeenCalledTimes(1); expect(execMock).toHaveBeenCalledTimes(1);
expect(execMock).toHaveBeenCalledWith(`"tar"`, args); expect(execMock).toHaveBeenCalledWith(`"tar"`, args);
expect(saveCacheMock).toHaveBeenCalledTimes(1); expect(saveCacheMock).toHaveBeenCalledTimes(1);
expect(saveCacheMock).toHaveBeenCalledWith(primaryKey, archivePath); expect(saveCacheMock).toHaveBeenCalledWith(cacheId, archivePath);
expect(logWarningMock).toHaveBeenCalledTimes(1); expect(logWarningMock).toHaveBeenCalledTimes(1);
expect(logWarningMock).toHaveBeenCalledWith("HTTP Error Occurred"); expect(logWarningMock).toHaveBeenCalledWith("HTTP Error Occurred");
@ -321,32 +329,40 @@ test("save with valid inputs uploads a cache", async () => {
const cachePath = path.resolve(inputPath); const cachePath = path.resolve(inputPath);
testUtils.setInput(Inputs.Path, inputPath); testUtils.setInput(Inputs.Path, inputPath);
const cacheId = 4;
const reserveCacheMock = jest.spyOn(cacheHttpClient, "reserveCache").mockImplementationOnce(() => {
return Promise.resolve(cacheId);
});
const execMock = jest.spyOn(exec, "exec"); const execMock = jest.spyOn(exec, "exec");
const saveCacheMock = jest.spyOn(cacheHttpClient, "saveCache"); const saveCacheMock = jest.spyOn(cacheHttpClient, "saveCache");
await run(); await run();
expect(reserveCacheMock).toHaveBeenCalledTimes(1);
expect(reserveCacheMock).toHaveBeenCalledWith(primaryKey);
const archivePath = path.join("/foo/bar", "cache.tgz"); const archivePath = path.join("/foo/bar", "cache.tgz");
const IS_WINDOWS = process.platform === "win32"; const IS_WINDOWS = process.platform === "win32";
const args = IS_WINDOWS const args = IS_WINDOWS
? [ ? [
"-cz", "-cz",
"--force-local", "--force-local",
"-f", "-f",
archivePath.replace(/\\/g, "/"), archivePath.replace(/\\/g, "/"),
"-C", "-C",
cachePath.replace(/\\/g, "/"), cachePath.replace(/\\/g, "/"),
"." "."
] ]
: ["-cz", "-f", archivePath, "-C", cachePath, "."]; : ["-cz", "-f", archivePath, "-C", cachePath, "."];
expect(execMock).toHaveBeenCalledTimes(1); expect(execMock).toHaveBeenCalledTimes(1);
expect(execMock).toHaveBeenCalledWith(`"tar"`, args); expect(execMock).toHaveBeenCalledWith(`"tar"`, args);
expect(saveCacheMock).toHaveBeenCalledTimes(1); expect(saveCacheMock).toHaveBeenCalledTimes(1);
expect(saveCacheMock).toHaveBeenCalledWith(primaryKey, archivePath); expect(saveCacheMock).toHaveBeenCalledWith(cacheId, archivePath);
expect(failedMock).toHaveBeenCalledTimes(0); expect(failedMock).toHaveBeenCalledTimes(0);
}); });

101
src/cacheHttpClient.ts
Unescape Escape View File

@ -107,9 +107,7 @@ export async function downloadCache(
} }
// Reserve Cache // Reserve Cache
export async function reserveCache( export async function reserveCache(key: string): Promise<number> {
key: string
): Promise<number> {
const restClient = createRestClient(); const restClient = createRestClient();
const reserveCacheRequest: ReserveCacheRequest = { const reserveCacheRequest: ReserveCacheRequest = {
@ -133,14 +131,6 @@ function getContentRange(start: number, end: number): string {
return `bytes ${start}-${end}/*`; return `bytes ${start}-${end}/*`;
} }
// function bufferToStream(buffer: Buffer): NodeJS.ReadableStream {
// const stream = new Duplex();
// stream.push(buffer);
// stream.push(null);
// return stream;
// }
async function uploadChunk( async function uploadChunk(
restClient: RestClient, restClient: RestClient,
resourceUrl: string, resourceUrl: string,
@ -148,53 +138,74 @@ async function uploadChunk(
start: number, start: number,
end: number end: number
): Promise<IRestResponse<void>> { ): Promise<IRestResponse<void>> {
core.debug(`Uploading chunk of size ${end - start + 1} bytes at offset ${start} with content range: ${getContentRange(start, end)}`); core.debug(
`Uploading chunk of size ${end -
start +
1} bytes at offset ${start} with content range: ${getContentRange(
start,
end
)}`
);
const requestOptions = getRequestOptions(); const requestOptions = getRequestOptions();
requestOptions.additionalHeaders = { requestOptions.additionalHeaders = {
"Content-Type": "application/octet-stream", "Content-Type": "application/octet-stream",
"Content-Range": getContentRange(start, end) "Content-Range": getContentRange(start, end)
}; };
return await restClient.uploadStream<void>("PATCH", resourceUrl, data, requestOptions); return await restClient.uploadStream<void>(
} "PATCH",
resourceUrl,
async function commitCache( data,
restClient: RestClient,
cacheId: number,
filesize: number
): Promise<IRestResponse<void>> {
const requestOptions = getRequestOptions();
const commitCacheRequest: CommitCacheRequest = { size: filesize };
return await restClient.create(
`caches/${cacheId.toString()}`,
commitCacheRequest,
requestOptions requestOptions
); );
} }
async function uploadFile(restClient: RestClient, cacheId: number, archivePath: string): Promise<void> { async function uploadFile(
restClient: RestClient,
cacheId: number,
archivePath: string
): Promise<void> {
// Upload Chunks // Upload Chunks
const fileSize = fs.statSync(archivePath).size; const fileSize = fs.statSync(archivePath).size;
const resourceUrl = getCacheApiUrl() + "caches/" + cacheId.toString(); const resourceUrl = getCacheApiUrl() + "caches/" + cacheId.toString();
const responses: IRestResponse<void>[] = []; const responses: IRestResponse<void>[] = [];
const fd = fs.openSync(archivePath, "r"); const fd = fs.openSync(archivePath, "r");
const concurrency = 16; // # of HTTP requests in parallel const concurrency = 4; // # of HTTP requests in parallel
const MAX_CHUNK_SIZE = 32000000; // 32 MB Chunks const MAX_CHUNK_SIZE = 32000000; // 32 MB Chunks
core.debug(`Concurrency: ${concurrency} and Chunk Size: ${MAX_CHUNK_SIZE}`); core.debug(`Concurrency: ${concurrency} and Chunk Size: ${MAX_CHUNK_SIZE}`);
const parallelUploads = [...new Array(concurrency).keys()]; const parallelUploads = [...new Array(concurrency).keys()];
core.debug("Awaiting all uploads"); core.debug("Awaiting all uploads");
let offset = 0; let offset = 0;
await Promise.all(parallelUploads.map(async () => { await Promise.all(
while (offset < fileSize) { parallelUploads.map(async () => {
const chunkSize = offset + MAX_CHUNK_SIZE > fileSize ? fileSize - offset : MAX_CHUNK_SIZE; while (offset < fileSize) {
const start = offset; const chunkSize =
const end = offset + chunkSize - 1; offset + MAX_CHUNK_SIZE > fileSize
offset += MAX_CHUNK_SIZE; ? fileSize - offset
const chunk = fs.createReadStream(archivePath, { fd, start, end, autoClose: false }); : MAX_CHUNK_SIZE;
responses.push(await uploadChunk(restClient, resourceUrl, chunk, start, end)); const start = offset;
} const end = offset + chunkSize - 1;
})); offset += MAX_CHUNK_SIZE;
const chunk = fs.createReadStream(archivePath, {
fd,
start,
end,
autoClose: false
});
responses.push(
await uploadChunk(
restClient,
resourceUrl,
chunk,
start,
end
)
);
}
})
);
fs.closeSync(fd); fs.closeSync(fd);
@ -210,6 +221,20 @@ async function uploadFile(restClient: RestClient, cacheId: number, archivePath:
return; return;
} }
async function commitCache(
restClient: RestClient,
cacheId: number,
filesize: number
): Promise<IRestResponse<void>> {
const requestOptions = getRequestOptions();
const commitCacheRequest: CommitCacheRequest = { size: filesize };
return await restClient.create(
`caches/${cacheId.toString()}`,
commitCacheRequest,
requestOptions
);
}
export async function saveCache( export async function saveCache(
cacheId: number, cacheId: number,
archivePath: string archivePath: string
@ -219,8 +244,8 @@ export async function saveCache(
core.debug("Upload cache"); core.debug("Upload cache");
await uploadFile(restClient, cacheId, archivePath); await uploadFile(restClient, cacheId, archivePath);
core.debug("Commiting cache");
// Commit Cache // Commit Cache
core.debug("Commiting cache");
const cacheSize = utils.getArchiveFileSize(archivePath); const cacheSize = utils.getArchiveFileSize(archivePath);
const commitCacheResponse = await commitCache( const commitCacheResponse = await commitCache(
restClient, restClient,

1
src/restore.ts
Unescape Escape View File

@ -82,7 +82,6 @@ async function run(): Promise<void> {
cacheEntry?.archiveLocation, cacheEntry?.archiveLocation,
archivePath archivePath
); );
await exec(`md5sum`, [archivePath]);
const archiveFileSize = utils.getArchiveFileSize(archivePath); const archiveFileSize = utils.getArchiveFileSize(archivePath);
core.info( core.info(

5
src/save.ts
Unescape Escape View File

@ -38,7 +38,9 @@ async function run(): Promise<void> {
core.debug("Reserving Cache"); core.debug("Reserving Cache");
const cacheId = await cacheHttpClient.reserveCache(primaryKey); const cacheId = await cacheHttpClient.reserveCache(primaryKey);
if (cacheId < 0) { if (cacheId < 0) {
core.info(`Unable to reserve cache with key ${primaryKey}, another job may be creating this cache.`); core.info(
`Unable to reserve cache with key ${primaryKey}, another job may be creating this cache.`
);
return; return;
} }
core.debug(`Cache ID: ${cacheId}`); core.debug(`Cache ID: ${cacheId}`);
@ -84,7 +86,6 @@ async function run(): Promise<void> {
return; return;
} }
await exec(`md5sum`, [archivePath]);
core.debug("Saving Cache"); core.debug("Saving Cache");
await cacheHttpClient.saveCache(cacheId, archivePath); await cacheHttpClient.saveCache(cacheId, archivePath);
} catch (error) { } catch (error) {

Write Preview
Loading…
Cancel
Save