actions
/
checkout
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

.

Browse Source
users/ericsciple/m163tarball_efficient_download
eric sciple 5 years ago
parent d79ea53307
commit 093dbebc2e
2 changed files with 240 additions and 140 deletions
Show Stats Download Patch File Download Diff File

178
dist/index.js vendored
Unescape Escape View File

@ -8358,7 +8358,6 @@ const core = __importStar(__webpack_require__(470));
const exec = __importStar(__webpack_require__(986)); const exec = __importStar(__webpack_require__(986));
const fs = __importStar(__webpack_require__(747)); const fs = __importStar(__webpack_require__(747));
const github = __importStar(__webpack_require__(469)); const github = __importStar(__webpack_require__(469));
const https = __importStar(__webpack_require__(211));
const io = __importStar(__webpack_require__(1)); const io = __importStar(__webpack_require__(1));
const path = __importStar(__webpack_require__(622)); const path = __importStar(__webpack_require__(622));
const refHelper = __importStar(__webpack_require__(227)); const refHelper = __importStar(__webpack_require__(227));
@ -8371,28 +8370,44 @@ function downloadRepository(accessToken, owner, repo, ref, commit, repositoryPat
const runnerTemp = process.env['RUNNER_TEMP']; const runnerTemp = process.env['RUNNER_TEMP'];
assert.ok(runnerTemp, 'RUNNER_TEMP not defined'); assert.ok(runnerTemp, 'RUNNER_TEMP not defined');
const archivePath = path.join(runnerTemp, 'checkout.tar.gz'); const archivePath = path.join(runnerTemp, 'checkout.tar.gz');
// await fs.promises.writeFile(archivePath, raw) // Ensure file does not exist
// Get the archive URL using the REST API core.debug(`Ensuring archive file does not exist: ${archivePath}`);
yield retryHelper.execute(() => __awaiter(this, void 0, void 0, function* () {
// Prepare the archive stream
core.debug(`Preparing the archive stream: ${archivePath}`);
yield io.rmRF(archivePath); yield io.rmRF(archivePath);
const fileStream = fs.createWriteStream(archivePath);
const fileStreamClosed = getFileClosedPromise(fileStream);
try {
// Get the archive URL
core.info('Getting archive URL');
const archiveUrl = yield getArchiveUrl(accessToken, owner, repo, ref, commit);
// Download the archive // Download the archive
core.info('Downloading the archive'); // Do not print the archive URL because it has an embedded token let archiveData = yield retryHelper.execute(() => __awaiter(this, void 0, void 0, function* () {
yield downloadFile(archiveUrl, fileStream); core.info('Downloading the archive using the REST API');
} yield yield downloadArchive(accessToken, owner, repo, ref, commit);
finally {
fileStream.end();
yield fileStreamClosed;
}
// return Buffer.from(response.data) // response.data is ArrayBuffer
})); }));
// Write archive to disk
core.info('Writing archive to disk');
yield fs.promises.writeFile(archivePath, archiveData);
archiveData = undefined;
// // Get the archive URL using the REST API
// await retryHelper.execute(async () => {
// // Prepare the archive stream
// core.debug(`Preparing the archive stream: ${archivePath}`)
// await io.rmRF(archivePath)
// const fileStream = fs.createWriteStream(archivePath)
// const fileStreamClosed = getFileClosedPromise(fileStream)
// try {
// // Get the archive URL
// core.info('Getting archive URL')
// const archiveUrl = await getArchiveUrl(
// accessToken,
// owner,
// repo,
// ref,
// commit
// )
// // Download the archive
// core.info('Downloading the archive') // Do not print the archive URL because it has an embedded token
// await downloadFile(archiveUrl, fileStream)
// } finally {
// fileStream.end()
// await fileStreamClosed
// }
// })
// return Buffer.from(response.data) // response.data is ArrayBuffer
// // Download the archive // // Download the archive
// core.info('Downloading the archive') // Do not print the URL since it contains a token to download the archive // core.info('Downloading the archive') // Do not print the URL since it contains a token to download the archive
// await downloadFile(archiveUrl, archivePath) // await downloadFile(archiveUrl, archivePath)
@ -8443,11 +8458,10 @@ function downloadRepository(accessToken, owner, repo, ref, commit, repositoryPat
}); });
} }
exports.downloadRepository = downloadRepository; exports.downloadRepository = downloadRepository;
function getArchiveUrl(accessToken, owner, repo, ref, commit) { function downloadArchive(accessToken, owner, repo, ref, commit) {
return __awaiter(this, void 0, void 0, function* () { return __awaiter(this, void 0, void 0, function* () {
const octokit = new github.GitHub(accessToken); const octokit = new github.GitHub(accessToken);
const params = { const params = {
method: 'HEAD',
owner: owner, owner: owner,
repo: repo, repo: repo,
archive_format: IS_WINDOWS ? 'zipball' : 'tarball', archive_format: IS_WINDOWS ? 'zipball' : 'tarball',
@ -8457,51 +8471,87 @@ function getArchiveUrl(accessToken, owner, repo, ref, commit) {
console.log('GOT THE RESPONSE'); console.log('GOT THE RESPONSE');
console.log(`status=${response.status}`); console.log(`status=${response.status}`);
console.log(`headers=${JSON.stringify(response.headers)}`); console.log(`headers=${JSON.stringify(response.headers)}`);
console.log(`headers=${JSON.stringify(response.data)}`); console.log(`data=${JSON.stringify(response.data)}`);
if (response.status != 200) { if (response.status != 200) {
throw new Error(`Unexpected response from GitHub API. Status: '${response.status}'`); throw new Error(`Unexpected response from GitHub API. Status: '${response.status}'`);
} }
console.log('GETTING THE LOCATION'); return Buffer.from(response.data); // response.data is ArrayBuffer
const archiveUrl = response.headers['Location']; // Do not print the archive URL because it has an embedded token // console.log('GETTING THE LOCATION')
assert.ok(archiveUrl, `Expected GitHub API response to contain 'Location' header`); // const archiveUrl = response.headers['Location'] // Do not print the archive URL because it has an embedded token
return archiveUrl; // assert.ok(
}); // archiveUrl,
} // `Expected GitHub API response to contain 'Location' header`
function downloadFile(url, fileStream) { // )
return new Promise((resolve, reject) => { // return archiveUrl
try { });
https.get(url, (response) => { }
if (response.statusCode != 200) { // async function getArchiveUrl(
reject(`Request failed with status '${response.statusCode}'`); // accessToken: string,
response.resume(); // Consume response data to free up memory // owner: string,
return; // repo: string,
} // ref: string,
response.on('data', chunk => { // commit: string
fileStream.write(chunk); // ): Promise<string> {
}); // const octokit = new github.GitHub(accessToken)
response.on('end', () => { // const params: RequestOptions & ReposGetArchiveLinkParams = {
resolve(); // method: 'HEAD',
}); // owner: owner,
response.on('error', err => { // repo: repo,
reject(err); // archive_format: IS_WINDOWS ? 'zipball' : 'tarball',
}); // ref: refHelper.getDownloadRef(ref, commit)
}); // }
} // const response = await octokit.repos.getArchiveLink(params)
catch (err) { // console.log('GOT THE RESPONSE')
reject(err); // console.log(`status=${response.status}`)
} // console.log(`headers=${JSON.stringify(response.headers)}`)
}); // console.log(`data=${JSON.stringify(response.data)}`)
} // if (response.status != 200) {
function getFileClosedPromise(stream) { // throw new Error(
return new Promise((resolve, reject) => { // `Unexpected response from GitHub API. Status: '${response.status}'`
stream.on('error', err => { // )
reject(err); // }
}); // console.log('GETTING THE LOCATION')
stream.on('finish', () => { // const archiveUrl = response.headers['Location'] // Do not print the archive URL because it has an embedded token
resolve(); // assert.ok(
}); // archiveUrl,
}); // `Expected GitHub API response to contain 'Location' header`
} // )
// return archiveUrl
// }
// function downloadFile(url: string, fileStream: WriteStream): Promise<void> {
// return new Promise((resolve, reject) => {
// try {
// https.get(url, (response: IncomingMessage) => {
// if (response.statusCode != 200) {
// reject(`Request failed with status '${response.statusCode}'`)
// response.resume() // Consume response data to free up memory
// return
// }
// response.on('data', chunk => {
// fileStream.write(chunk)
// })
// response.on('end', () => {
// resolve()
// })
// response.on('error', err => {
// reject(err)
// })
// })
// } catch (err) {
// reject(err)
// }
// })
// }
// function getFileClosedPromise(stream: WriteStream): Promise<void> {
// return new Promise((resolve, reject) => {
// stream.on('error', err => {
// reject(err)
// })
// stream.on('finish', () => {
// resolve()
// })
// })
// }
/***/ }), /***/ }),

192
src/github-api-helper.ts
Unescape Escape View File

@ -28,37 +28,51 @@ export async function downloadRepository(
const runnerTemp = process.env['RUNNER_TEMP'] as string const runnerTemp = process.env['RUNNER_TEMP'] as string
assert.ok(runnerTemp, 'RUNNER_TEMP not defined') assert.ok(runnerTemp, 'RUNNER_TEMP not defined')
const archivePath = path.join(runnerTemp, 'checkout.tar.gz') const archivePath = path.join(runnerTemp, 'checkout.tar.gz')
// await fs.promises.writeFile(archivePath, raw)
// Get the archive URL using the REST API // Ensure file does not exist
await retryHelper.execute(async () => { core.debug(`Ensuring archive file does not exist: ${archivePath}`)
// Prepare the archive stream
core.debug(`Preparing the archive stream: ${archivePath}`)
await io.rmRF(archivePath) await io.rmRF(archivePath)
const fileStream = fs.createWriteStream(archivePath)
const fileStreamClosed = getFileClosedPromise(fileStream)
try {
// Get the archive URL
core.info('Getting archive URL')
const archiveUrl = await getArchiveUrl(
accessToken,
owner,
repo,
ref,
commit
)
// Download the archive // Download the archive
core.info('Downloading the archive') // Do not print the archive URL because it has an embedded token let archiveData = await retryHelper.execute(async () => {
await downloadFile(archiveUrl, fileStream) core.info('Downloading the archive using the REST API')
} finally { await await downloadArchive(accessToken, owner, repo, ref, commit)
fileStream.end() })
await fileStreamClosed
} // Write archive to disk
core.info('Writing archive to disk')
await fs.promises.writeFile(archivePath, archiveData)
archiveData = undefined
// // Get the archive URL using the REST API
// await retryHelper.execute(async () => {
// // Prepare the archive stream
// core.debug(`Preparing the archive stream: ${archivePath}`)
// await io.rmRF(archivePath)
// const fileStream = fs.createWriteStream(archivePath)
// const fileStreamClosed = getFileClosedPromise(fileStream)
// try {
// // Get the archive URL
// core.info('Getting archive URL')
// const archiveUrl = await getArchiveUrl(
// accessToken,
// owner,
// repo,
// ref,
// commit
// )
// // Download the archive
// core.info('Downloading the archive') // Do not print the archive URL because it has an embedded token
// await downloadFile(archiveUrl, fileStream)
// } finally {
// fileStream.end()
// await fileStreamClosed
// }
// })
// return Buffer.from(response.data) // response.data is ArrayBuffer // return Buffer.from(response.data) // response.data is ArrayBuffer
})
// // Download the archive // // Download the archive
// core.info('Downloading the archive') // Do not print the URL since it contains a token to download the archive // core.info('Downloading the archive') // Do not print the URL since it contains a token to download the archive
@ -120,16 +134,15 @@ export async function downloadRepository(
} as ExecOptions) } as ExecOptions)
} }
async function getArchiveUrl( async function downloadArchive(
accessToken: string, accessToken: string,
owner: string, owner: string,
repo: string, repo: string,
ref: string, ref: string,
commit: string commit: string
): Promise<string> { ): Promise<Buffer> {
const octokit = new github.GitHub(accessToken) const octokit = new github.GitHub(accessToken)
const params: RequestOptions & ReposGetArchiveLinkParams = { const params: ReposGetArchiveLinkParams = {
method: 'HEAD',
owner: owner, owner: owner,
repo: repo, repo: repo,
archive_format: IS_WINDOWS ? 'zipball' : 'tarball', archive_format: IS_WINDOWS ? 'zipball' : 'tarball',
@ -139,54 +152,91 @@ async function getArchiveUrl(
console.log('GOT THE RESPONSE') console.log('GOT THE RESPONSE')
console.log(`status=${response.status}`) console.log(`status=${response.status}`)
console.log(`headers=${JSON.stringify(response.headers)}`) console.log(`headers=${JSON.stringify(response.headers)}`)
console.log(`headers=${JSON.stringify(response.data)}`) console.log(`data=${JSON.stringify(response.data)}`)
if (response.status != 200) { if (response.status != 200) {
throw new Error( throw new Error(
`Unexpected response from GitHub API. Status: '${response.status}'` `Unexpected response from GitHub API. Status: '${response.status}'`
) )
} }
console.log('GETTING THE LOCATION')
const archiveUrl = response.headers['Location'] // Do not print the archive URL because it has an embedded token
assert.ok(
archiveUrl,
`Expected GitHub API response to contain 'Location' header`
)
return archiveUrl
}
function downloadFile(url: string, fileStream: WriteStream): Promise<void> { return Buffer.from(response.data) // response.data is ArrayBuffer
return new Promise((resolve, reject) => {
try {
https.get(url, (response: IncomingMessage) => {
if (response.statusCode != 200) {
reject(`Request failed with status '${response.statusCode}'`)
response.resume() // Consume response data to free up memory
return
}
response.on('data', chunk => { // console.log('GETTING THE LOCATION')
fileStream.write(chunk) // const archiveUrl = response.headers['Location'] // Do not print the archive URL because it has an embedded token
}) // assert.ok(
response.on('end', () => { // archiveUrl,
resolve() // `Expected GitHub API response to contain 'Location' header`
}) // )
response.on('error', err => { // return archiveUrl
reject(err)
})
})
} catch (err) {
reject(err)
}
})
} }
function getFileClosedPromise(stream: WriteStream): Promise<void> { // async function getArchiveUrl(
return new Promise((resolve, reject) => { // accessToken: string,
stream.on('error', err => { // owner: string,
reject(err) // repo: string,
}) // ref: string,
stream.on('finish', () => { // commit: string
resolve() // ): Promise<string> {
}) // const octokit = new github.GitHub(accessToken)
}) // const params: RequestOptions & ReposGetArchiveLinkParams = {
} // method: 'HEAD',
// owner: owner,
// repo: repo,
// archive_format: IS_WINDOWS ? 'zipball' : 'tarball',
// ref: refHelper.getDownloadRef(ref, commit)
// }
// const response = await octokit.repos.getArchiveLink(params)
// console.log('GOT THE RESPONSE')
// console.log(`status=${response.status}`)
// console.log(`headers=${JSON.stringify(response.headers)}`)
// console.log(`data=${JSON.stringify(response.data)}`)
// if (response.status != 200) {
// throw new Error(
// `Unexpected response from GitHub API. Status: '${response.status}'`
// )
// }
// console.log('GETTING THE LOCATION')
// const archiveUrl = response.headers['Location'] // Do not print the archive URL because it has an embedded token
// assert.ok(
// archiveUrl,
// `Expected GitHub API response to contain 'Location' header`
// )
// return archiveUrl
// }
// function downloadFile(url: string, fileStream: WriteStream): Promise<void> {
// return new Promise((resolve, reject) => {
// try {
// https.get(url, (response: IncomingMessage) => {
// if (response.statusCode != 200) {
// reject(`Request failed with status '${response.statusCode}'`)
// response.resume() // Consume response data to free up memory
// return
// }
// response.on('data', chunk => {
// fileStream.write(chunk)
// })
// response.on('end', () => {
// resolve()
// })
// response.on('error', err => {
// reject(err)
// })
// })
// } catch (err) {
// reject(err)
// }
// })
// }
// function getFileClosedPromise(stream: WriteStream): Promise<void> {
// return new Promise((resolve, reject) => {
// stream.on('error', err => {
// reject(err)
// })
// stream.on('finish', () => {
// resolve()
// })
// })
// }

Write Preview
Loading…
Cancel
Save