From cdec5dec0db8456760a3d6b1a5d6f7c34f1f7fdd Mon Sep 17 00:00:00 2001 From: Josh Gross Date: Wed, 18 Dec 2019 10:59:51 -0500 Subject: [PATCH] Format and update tests --- __tests__/save.test.ts | 68 ++++++++++++++++----------- src/cacheHttpClient.ts | 101 +++++++++++++++++++++++++---------------- src/restore.ts | 1 - src/save.ts | 5 +- 4 files changed, 108 insertions(+), 67 deletions(-) diff --git a/__tests__/save.test.ts b/__tests__/save.test.ts index b0eb462..221e722 100644 --- a/__tests__/save.test.ts +++ b/__tests__/save.test.ts @@ -212,14 +212,14 @@ test("save with large cache outputs warning", async () => { const IS_WINDOWS = process.platform === "win32"; const args = IS_WINDOWS ? [ - "-cz", - "--force-local", - "-f", - archivePath.replace(/\\/g, "/"), - "-C", - cachePath.replace(/\\/g, "/"), - "." - ] + "-cz", + "--force-local", + "-f", + archivePath.replace(/\\/g, "/"), + "-C", + cachePath.replace(/\\/g, "/"), + "." + ] : ["-cz", "-f", archivePath, "-C", cachePath, "."]; expect(execMock).toHaveBeenCalledTimes(1); @@ -259,6 +259,11 @@ test("save with server error outputs warning", async () => { const cachePath = path.resolve(inputPath); testUtils.setInput(Inputs.Path, inputPath); + const cacheId = 4; + const reserveCacheMock = jest.spyOn(cacheHttpClient, "reserveCache").mockImplementationOnce(() => { + return Promise.resolve(cacheId); + }); + const execMock = jest.spyOn(exec, "exec"); const saveCacheMock = jest @@ -269,26 +274,29 @@ test("save with server error outputs warning", async () => { await run(); + expect(reserveCacheMock).toHaveBeenCalledTimes(1); + expect(reserveCacheMock).toHaveBeenCalledWith(primaryKey); + const archivePath = path.join("/foo/bar", "cache.tgz"); const IS_WINDOWS = process.platform === "win32"; const args = IS_WINDOWS ? [ - "-cz", - "--force-local", - "-f", - archivePath.replace(/\\/g, "/"), - "-C", - cachePath.replace(/\\/g, "/"), - "." - ] + "-cz", + "--force-local", + "-f", + archivePath.replace(/\\/g, "/"), + "-C", + cachePath.replace(/\\/g, "/"), + "." + ] : ["-cz", "-f", archivePath, "-C", cachePath, "."]; expect(execMock).toHaveBeenCalledTimes(1); expect(execMock).toHaveBeenCalledWith(`"tar"`, args); expect(saveCacheMock).toHaveBeenCalledTimes(1); - expect(saveCacheMock).toHaveBeenCalledWith(primaryKey, archivePath); + expect(saveCacheMock).toHaveBeenCalledWith(cacheId, archivePath); expect(logWarningMock).toHaveBeenCalledTimes(1); expect(logWarningMock).toHaveBeenCalledWith("HTTP Error Occurred"); @@ -321,32 +329,40 @@ test("save with valid inputs uploads a cache", async () => { const cachePath = path.resolve(inputPath); testUtils.setInput(Inputs.Path, inputPath); + const cacheId = 4; + const reserveCacheMock = jest.spyOn(cacheHttpClient, "reserveCache").mockImplementationOnce(() => { + return Promise.resolve(cacheId); + }); + const execMock = jest.spyOn(exec, "exec"); const saveCacheMock = jest.spyOn(cacheHttpClient, "saveCache"); await run(); + expect(reserveCacheMock).toHaveBeenCalledTimes(1); + expect(reserveCacheMock).toHaveBeenCalledWith(primaryKey); + const archivePath = path.join("/foo/bar", "cache.tgz"); const IS_WINDOWS = process.platform === "win32"; const args = IS_WINDOWS ? [ - "-cz", - "--force-local", - "-f", - archivePath.replace(/\\/g, "/"), - "-C", - cachePath.replace(/\\/g, "/"), - "." - ] + "-cz", + "--force-local", + "-f", + archivePath.replace(/\\/g, "/"), + "-C", + cachePath.replace(/\\/g, "/"), + "." + ] : ["-cz", "-f", archivePath, "-C", cachePath, "."]; expect(execMock).toHaveBeenCalledTimes(1); expect(execMock).toHaveBeenCalledWith(`"tar"`, args); expect(saveCacheMock).toHaveBeenCalledTimes(1); - expect(saveCacheMock).toHaveBeenCalledWith(primaryKey, archivePath); + expect(saveCacheMock).toHaveBeenCalledWith(cacheId, archivePath); expect(failedMock).toHaveBeenCalledTimes(0); }); diff --git a/src/cacheHttpClient.ts b/src/cacheHttpClient.ts index 24a22dc..9501762 100644 --- a/src/cacheHttpClient.ts +++ b/src/cacheHttpClient.ts @@ -107,9 +107,7 @@ export async function downloadCache( } // Reserve Cache -export async function reserveCache( - key: string -): Promise { +export async function reserveCache(key: string): Promise { const restClient = createRestClient(); const reserveCacheRequest: ReserveCacheRequest = { @@ -133,14 +131,6 @@ function getContentRange(start: number, end: number): string { return `bytes ${start}-${end}/*`; } -// function bufferToStream(buffer: Buffer): NodeJS.ReadableStream { -// const stream = new Duplex(); -// stream.push(buffer); -// stream.push(null); - -// return stream; -// } - async function uploadChunk( restClient: RestClient, resourceUrl: string, @@ -148,53 +138,74 @@ async function uploadChunk( start: number, end: number ): Promise> { - core.debug(`Uploading chunk of size ${end - start + 1} bytes at offset ${start} with content range: ${getContentRange(start, end)}`); + core.debug( + `Uploading chunk of size ${end - + start + + 1} bytes at offset ${start} with content range: ${getContentRange( + start, + end + )}` + ); const requestOptions = getRequestOptions(); requestOptions.additionalHeaders = { "Content-Type": "application/octet-stream", "Content-Range": getContentRange(start, end) }; - return await restClient.uploadStream("PATCH", resourceUrl, data, requestOptions); -} - -async function commitCache( - restClient: RestClient, - cacheId: number, - filesize: number -): Promise> { - const requestOptions = getRequestOptions(); - const commitCacheRequest: CommitCacheRequest = { size: filesize }; - return await restClient.create( - `caches/${cacheId.toString()}`, - commitCacheRequest, + return await restClient.uploadStream( + "PATCH", + resourceUrl, + data, requestOptions ); } -async function uploadFile(restClient: RestClient, cacheId: number, archivePath: string): Promise { +async function uploadFile( + restClient: RestClient, + cacheId: number, + archivePath: string +): Promise { // Upload Chunks const fileSize = fs.statSync(archivePath).size; const resourceUrl = getCacheApiUrl() + "caches/" + cacheId.toString(); const responses: IRestResponse[] = []; const fd = fs.openSync(archivePath, "r"); - const concurrency = 16; // # of HTTP requests in parallel + const concurrency = 4; // # of HTTP requests in parallel const MAX_CHUNK_SIZE = 32000000; // 32 MB Chunks core.debug(`Concurrency: ${concurrency} and Chunk Size: ${MAX_CHUNK_SIZE}`); + const parallelUploads = [...new Array(concurrency).keys()]; core.debug("Awaiting all uploads"); let offset = 0; - await Promise.all(parallelUploads.map(async () => { - while (offset < fileSize) { - const chunkSize = offset + MAX_CHUNK_SIZE > fileSize ? fileSize - offset : MAX_CHUNK_SIZE; - const start = offset; - const end = offset + chunkSize - 1; - offset += MAX_CHUNK_SIZE; - const chunk = fs.createReadStream(archivePath, { fd, start, end, autoClose: false }); - responses.push(await uploadChunk(restClient, resourceUrl, chunk, start, end)); - } - })); + await Promise.all( + parallelUploads.map(async () => { + while (offset < fileSize) { + const chunkSize = + offset + MAX_CHUNK_SIZE > fileSize + ? fileSize - offset + : MAX_CHUNK_SIZE; + const start = offset; + const end = offset + chunkSize - 1; + offset += MAX_CHUNK_SIZE; + const chunk = fs.createReadStream(archivePath, { + fd, + start, + end, + autoClose: false + }); + responses.push( + await uploadChunk( + restClient, + resourceUrl, + chunk, + start, + end + ) + ); + } + }) + ); fs.closeSync(fd); @@ -210,6 +221,20 @@ async function uploadFile(restClient: RestClient, cacheId: number, archivePath: return; } +async function commitCache( + restClient: RestClient, + cacheId: number, + filesize: number +): Promise> { + const requestOptions = getRequestOptions(); + const commitCacheRequest: CommitCacheRequest = { size: filesize }; + return await restClient.create( + `caches/${cacheId.toString()}`, + commitCacheRequest, + requestOptions + ); +} + export async function saveCache( cacheId: number, archivePath: string @@ -219,8 +244,8 @@ export async function saveCache( core.debug("Upload cache"); await uploadFile(restClient, cacheId, archivePath); - core.debug("Commiting cache"); // Commit Cache + core.debug("Commiting cache"); const cacheSize = utils.getArchiveFileSize(archivePath); const commitCacheResponse = await commitCache( restClient, diff --git a/src/restore.ts b/src/restore.ts index 562d455..09a3d2f 100644 --- a/src/restore.ts +++ b/src/restore.ts @@ -82,7 +82,6 @@ async function run(): Promise { cacheEntry?.archiveLocation, archivePath ); - await exec(`md5sum`, [archivePath]); const archiveFileSize = utils.getArchiveFileSize(archivePath); core.info( diff --git a/src/save.ts b/src/save.ts index 78f5733..418d698 100644 --- a/src/save.ts +++ b/src/save.ts @@ -38,7 +38,9 @@ async function run(): Promise { core.debug("Reserving Cache"); const cacheId = await cacheHttpClient.reserveCache(primaryKey); if (cacheId < 0) { - core.info(`Unable to reserve cache with key ${primaryKey}, another job may be creating this cache.`); + core.info( + `Unable to reserve cache with key ${primaryKey}, another job may be creating this cache.` + ); return; } core.debug(`Cache ID: ${cacheId}`); @@ -84,7 +86,6 @@ async function run(): Promise { return; } - await exec(`md5sum`, [archivePath]); core.debug("Saving Cache"); await cacheHttpClient.saveCache(cacheId, archivePath); } catch (error) {