Compare commits

..

28 Commits

Author SHA1 Message Date
cdec5dec0d Format and update tests 2019-12-18 10:59:51 -05:00
2ce22df8c4 Test out 16 concurrency with 32mb chunks 2019-12-17 17:52:59 -05:00
8c77f01f0b Test out 16 concurrent requests 2019-12-17 17:35:30 -05:00
4fcbc07edb Test out higher concurrency limits 2019-12-17 17:26:05 -05:00
16019b42a9 Fix threads array 2019-12-17 17:19:16 -05:00
73a15dc5a9 Add more debug logging 2019-12-17 17:13:37 -05:00
574cd74b58 ? 2019-12-17 17:08:48 -05:00
14055801c2 Add missing await 2019-12-17 17:01:57 -05:00
289c5d2518 Concurrency take 2 2019-12-17 16:59:18 -05:00
ba6476e454 Bad implementation of parallel await 2019-12-17 16:17:55 -05:00
b425e87f79 Don't autoclose file 2019-12-17 15:46:56 -05:00
83f86c103f Make uploads serial 2019-12-17 15:43:50 -05:00
64668e22dd Move hashsum after tar creation 2019-12-17 15:11:47 -05:00
1c77f64ab3 Use correct hashing program 2019-12-17 15:05:04 -05:00
a70833fb48 Add debug hashing (won't work on windows) 2019-12-17 14:57:36 -05:00
b25804d19e Fix resource URLs 2019-12-17 14:52:40 -05:00
577b274c51 More debugging 2019-12-17 14:47:57 -05:00
0816faf84c Use fs streams directly 2019-12-17 14:40:24 -05:00
131e247bd2 Change to on end 2019-12-17 14:22:32 -05:00
2cbd952179 Add more debugging 2019-12-17 14:16:15 -05:00
994e3b75fc Add request header and debug statements 2019-12-17 14:10:58 -05:00
21dc9a47e6 Add release files 2019-12-17 13:56:42 -05:00
436418ea07 Separate out reserve call 2019-12-17 13:56:10 -05:00
7f6523f535 Linting tests 2019-12-13 15:39:29 -05:00
4d3086b6b8 Fix download cache entry tests 2019-12-13 15:37:15 -05:00
d788427754 Linting 2019-12-13 15:33:33 -05:00
a8adbe4b05 Fix cacheEntry type 2019-12-13 15:32:00 -05:00
bad827c28e Initial pass at chunked upload apis 2019-12-13 15:19:25 -05:00
17 changed files with 6179 additions and 1969 deletions

View File

@ -19,7 +19,6 @@ jobs:
strategy: strategy:
matrix: matrix:
os: [ubuntu-latest, windows-latest, macOS-latest] os: [ubuntu-latest, windows-latest, macOS-latest]
fail-fast: false
runs-on: ${{ matrix.os }} runs-on: ${{ matrix.os }}

3
.gitignore vendored
View File

@ -94,6 +94,3 @@ typings/
# DynamoDB Local files # DynamoDB Local files
.dynamodb/ .dynamodb/
# Text editor files
.vscode/

View File

@ -1,6 +1,6 @@
# cache # cache
This action allows caching dependencies and build outputs to improve workflow execution time. This GitHub Action allows caching dependencies and build outputs to improve workflow execution time.
<a href="https://github.com/actions/cache/actions?query=workflow%3ATests"><img alt="GitHub Actions status" src="https://github.com/actions/cache/workflows/Tests/badge.svg?branch=master&event=push"></a> <a href="https://github.com/actions/cache/actions?query=workflow%3ATests"><img alt="GitHub Actions status" src="https://github.com/actions/cache/workflows/Tests/badge.svg?branch=master&event=push"></a>
@ -56,31 +56,28 @@ jobs:
## Implementation Examples ## Implementation Examples
Every programming language and framework has its own way of caching. Every programming language and framework has it's own way of caching.
See [Examples](examples.md) for a list of `actions/cache` implementations for use with: See [Examples](examples.md) for a list of `actions/cache` implementations for use with:
- [C# - Nuget](./examples.md#c---nuget) - [C# - Nuget](./examples.md#c---nuget)
- [Elixir - Mix](./examples.md#elixir---mix) - [Elixir - Mix](./examples.md#elixir---mix)
- [Go - Modules](./examples.md#go---modules) - [Go - Modules](./examples.md#go---modules)
- [Haskell - Cabal](./examples.md#haskell---cabal)
- [Java - Gradle](./examples.md#java---gradle) - [Java - Gradle](./examples.md#java---gradle)
- [Java - Maven](./examples.md#java---maven) - [Java - Maven](./examples.md#java---maven)
- [Node - npm](./examples.md#node---npm) - [Node - npm](./examples.md#node---npm)
- [Node - Yarn](./examples.md#node---yarn) - [Node - Yarn](./examples.md#node---yarn)
- [PHP - Composer](./examples.md#php---composer) - [PHP - Composer](./examples.md#php---composer)
- [Python - pip](./examples.md#python---pip) - [Python - pip](./examples.md#python---pip)
- [R - renv](./examples.md#r---renv) - [Ruby - Gem](./examples.md#ruby---gem)
- [Ruby - Bundler](./examples.md#ruby---bundler)
- [Rust - Cargo](./examples.md#rust---cargo) - [Rust - Cargo](./examples.md#rust---cargo)
- [Scala - SBT](./examples.md#scala---sbt)
- [Swift, Objective-C - Carthage](./examples.md#swift-objective-c---carthage) - [Swift, Objective-C - Carthage](./examples.md#swift-objective-c---carthage)
- [Swift, Objective-C - CocoaPods](./examples.md#swift-objective-c---cocoapods) - [Swift, Objective-C - CocoaPods](./examples.md#swift-objective-c---cocoapods)
- [Swift - Swift Package Manager](./examples.md#swift---swift-package-manager)
## Cache Limits ## Cache Limits
A repository can have up to 5GB of caches. Once the 5GB limit is reached, older caches will be evicted based on when the cache was last accessed. Caches that are not accessed within the last week will also be evicted. Individual caches are limited to 400MB and a repository can have up to 2GB of caches. Once the 2GB limit is reached, older caches will be evicted based on when the cache was last accessed. Caches that are not accessed within the last week will also be evicted.
## Skipping steps based on cache-hit ## Skipping steps based on cache-hit

View File

@ -1,16 +1,18 @@
import * as core from "@actions/core"; import * as core from "@actions/core";
import * as exec from "@actions/exec";
import * as io from "@actions/io";
import * as path from "path"; import * as path from "path";
import * as cacheHttpClient from "../src/cacheHttpClient"; import * as cacheHttpClient from "../src/cacheHttpClient";
import { Events, Inputs } from "../src/constants"; import { Events, Inputs } from "../src/constants";
import { ArtifactCacheEntry } from "../src/contracts"; import { ArtifactCacheEntry } from "../src/contracts";
import run from "../src/restore"; import run from "../src/restore";
import * as tar from "../src/tar";
import * as actionUtils from "../src/utils/actionUtils"; import * as actionUtils from "../src/utils/actionUtils";
import * as testUtils from "../src/utils/testUtils"; import * as testUtils from "../src/utils/testUtils";
jest.mock("../src/cacheHttpClient"); jest.mock("@actions/exec");
jest.mock("../src/tar"); jest.mock("@actions/io");
jest.mock("../src/utils/actionUtils"); jest.mock("../src/utils/actionUtils");
jest.mock("../src/cacheHttpClient");
beforeAll(() => { beforeAll(() => {
jest.spyOn(actionUtils, "resolvePath").mockImplementation(filePath => { jest.spyOn(actionUtils, "resolvePath").mockImplementation(filePath => {
@ -33,6 +35,10 @@ beforeAll(() => {
const actualUtils = jest.requireActual("../src/utils/actionUtils"); const actualUtils = jest.requireActual("../src/utils/actionUtils");
return actualUtils.getSupportedEvents(); return actualUtils.getSupportedEvents();
}); });
jest.spyOn(io, "which").mockImplementation(tool => {
return Promise.resolve(tool);
});
}); });
beforeEach(() => { beforeEach(() => {
@ -239,7 +245,8 @@ test("restore with cache found", async () => {
.spyOn(actionUtils, "getArchiveFileSize") .spyOn(actionUtils, "getArchiveFileSize")
.mockReturnValue(fileSize); .mockReturnValue(fileSize);
const extractTarMock = jest.spyOn(tar, "extractTar"); const mkdirMock = jest.spyOn(io, "mkdirP");
const execMock = jest.spyOn(exec, "exec");
const setCacheHitOutputMock = jest.spyOn(actionUtils, "setCacheHitOutput"); const setCacheHitOutputMock = jest.spyOn(actionUtils, "setCacheHitOutput");
await run(); await run();
@ -253,9 +260,22 @@ test("restore with cache found", async () => {
archivePath archivePath
); );
expect(getArchiveFileSizeMock).toHaveBeenCalledWith(archivePath); expect(getArchiveFileSizeMock).toHaveBeenCalledWith(archivePath);
expect(mkdirMock).toHaveBeenCalledWith(cachePath);
expect(extractTarMock).toHaveBeenCalledTimes(1); const IS_WINDOWS = process.platform === "win32";
expect(extractTarMock).toHaveBeenCalledWith(archivePath, cachePath); const args = IS_WINDOWS
? [
"-xz",
"--force-local",
"-f",
archivePath.replace(/\\/g, "/"),
"-C",
cachePath.replace(/\\/g, "/")
]
: ["-xz", "-f", archivePath, "-C", cachePath];
expect(execMock).toHaveBeenCalledTimes(1);
expect(execMock).toHaveBeenCalledWith(`"tar"`, args);
expect(setCacheHitOutputMock).toHaveBeenCalledTimes(1); expect(setCacheHitOutputMock).toHaveBeenCalledTimes(1);
expect(setCacheHitOutputMock).toHaveBeenCalledWith(true); expect(setCacheHitOutputMock).toHaveBeenCalledWith(true);
@ -306,7 +326,8 @@ test("restore with a pull request event and cache found", async () => {
.spyOn(actionUtils, "getArchiveFileSize") .spyOn(actionUtils, "getArchiveFileSize")
.mockReturnValue(fileSize); .mockReturnValue(fileSize);
const extractTarMock = jest.spyOn(tar, "extractTar"); const mkdirMock = jest.spyOn(io, "mkdirP");
const execMock = jest.spyOn(exec, "exec");
const setCacheHitOutputMock = jest.spyOn(actionUtils, "setCacheHitOutput"); const setCacheHitOutputMock = jest.spyOn(actionUtils, "setCacheHitOutput");
await run(); await run();
@ -321,9 +342,22 @@ test("restore with a pull request event and cache found", async () => {
); );
expect(getArchiveFileSizeMock).toHaveBeenCalledWith(archivePath); expect(getArchiveFileSizeMock).toHaveBeenCalledWith(archivePath);
expect(infoMock).toHaveBeenCalledWith(`Cache Size: ~60 MB (62915000 B)`); expect(infoMock).toHaveBeenCalledWith(`Cache Size: ~60 MB (62915000 B)`);
expect(mkdirMock).toHaveBeenCalledWith(cachePath);
expect(extractTarMock).toHaveBeenCalledTimes(1); const IS_WINDOWS = process.platform === "win32";
expect(extractTarMock).toHaveBeenCalledWith(archivePath, cachePath); const args = IS_WINDOWS
? [
"-xz",
"--force-local",
"-f",
archivePath.replace(/\\/g, "/"),
"-C",
cachePath.replace(/\\/g, "/")
]
: ["-xz", "-f", archivePath, "-C", cachePath];
expect(execMock).toHaveBeenCalledTimes(1);
expect(execMock).toHaveBeenCalledWith(`"tar"`, args);
expect(setCacheHitOutputMock).toHaveBeenCalledTimes(1); expect(setCacheHitOutputMock).toHaveBeenCalledTimes(1);
expect(setCacheHitOutputMock).toHaveBeenCalledWith(true); expect(setCacheHitOutputMock).toHaveBeenCalledWith(true);
@ -374,7 +408,8 @@ test("restore with cache found for restore key", async () => {
.spyOn(actionUtils, "getArchiveFileSize") .spyOn(actionUtils, "getArchiveFileSize")
.mockReturnValue(fileSize); .mockReturnValue(fileSize);
const extractTarMock = jest.spyOn(tar, "extractTar"); const mkdirMock = jest.spyOn(io, "mkdirP");
const execMock = jest.spyOn(exec, "exec");
const setCacheHitOutputMock = jest.spyOn(actionUtils, "setCacheHitOutput"); const setCacheHitOutputMock = jest.spyOn(actionUtils, "setCacheHitOutput");
await run(); await run();
@ -389,9 +424,22 @@ test("restore with cache found for restore key", async () => {
); );
expect(getArchiveFileSizeMock).toHaveBeenCalledWith(archivePath); expect(getArchiveFileSizeMock).toHaveBeenCalledWith(archivePath);
expect(infoMock).toHaveBeenCalledWith(`Cache Size: ~0 MB (142 B)`); expect(infoMock).toHaveBeenCalledWith(`Cache Size: ~0 MB (142 B)`);
expect(mkdirMock).toHaveBeenCalledWith(cachePath);
expect(extractTarMock).toHaveBeenCalledTimes(1); const IS_WINDOWS = process.platform === "win32";
expect(extractTarMock).toHaveBeenCalledWith(archivePath, cachePath); const args = IS_WINDOWS
? [
"-xz",
"--force-local",
"-f",
archivePath.replace(/\\/g, "/"),
"-C",
cachePath.replace(/\\/g, "/")
]
: ["-xz", "-f", archivePath, "-C", cachePath];
expect(execMock).toHaveBeenCalledTimes(1);
expect(execMock).toHaveBeenCalledWith(`"tar"`, args);
expect(setCacheHitOutputMock).toHaveBeenCalledTimes(1); expect(setCacheHitOutputMock).toHaveBeenCalledTimes(1);
expect(setCacheHitOutputMock).toHaveBeenCalledWith(false); expect(setCacheHitOutputMock).toHaveBeenCalledWith(false);

View File

@ -1,17 +1,19 @@
import * as core from "@actions/core"; import * as core from "@actions/core";
import * as exec from "@actions/exec";
import * as io from "@actions/io";
import * as path from "path"; import * as path from "path";
import * as cacheHttpClient from "../src/cacheHttpClient"; import * as cacheHttpClient from "../src/cacheHttpClient";
import { Events, Inputs } from "../src/constants"; import { Events, Inputs } from "../src/constants";
import { ArtifactCacheEntry } from "../src/contracts"; import { ArtifactCacheEntry } from "../src/contracts";
import run from "../src/save"; import run from "../src/save";
import * as tar from "../src/tar";
import * as actionUtils from "../src/utils/actionUtils"; import * as actionUtils from "../src/utils/actionUtils";
import * as testUtils from "../src/utils/testUtils"; import * as testUtils from "../src/utils/testUtils";
jest.mock("@actions/core"); jest.mock("@actions/core");
jest.mock("../src/cacheHttpClient"); jest.mock("@actions/exec");
jest.mock("../src/tar"); jest.mock("@actions/io");
jest.mock("../src/utils/actionUtils"); jest.mock("../src/utils/actionUtils");
jest.mock("../src/cacheHttpClient");
beforeAll(() => { beforeAll(() => {
jest.spyOn(core, "getInput").mockImplementation((name, options) => { jest.spyOn(core, "getInput").mockImplementation((name, options) => {
@ -47,6 +49,10 @@ beforeAll(() => {
jest.spyOn(actionUtils, "createTempDirectory").mockImplementation(() => { jest.spyOn(actionUtils, "createTempDirectory").mockImplementation(() => {
return Promise.resolve("/foo/bar"); return Promise.resolve("/foo/bar");
}); });
jest.spyOn(io, "which").mockImplementation(tool => {
return Promise.resolve(tool);
});
}); });
beforeEach(() => { beforeEach(() => {
@ -122,7 +128,7 @@ test("save with exact match returns early", async () => {
return primaryKey; return primaryKey;
}); });
const createTarMock = jest.spyOn(tar, "createTar"); const execMock = jest.spyOn(exec, "exec");
await run(); await run();
@ -130,7 +136,7 @@ test("save with exact match returns early", async () => {
`Cache hit occurred on the primary key ${primaryKey}, not saving cache.` `Cache hit occurred on the primary key ${primaryKey}, not saving cache.`
); );
expect(createTarMock).toHaveBeenCalledTimes(0); expect(execMock).toHaveBeenCalledTimes(0);
expect(failedMock).toHaveBeenCalledTimes(0); expect(failedMock).toHaveBeenCalledTimes(0);
}); });
@ -192,9 +198,9 @@ test("save with large cache outputs warning", async () => {
const cachePath = path.resolve(inputPath); const cachePath = path.resolve(inputPath);
testUtils.setInput(Inputs.Path, inputPath); testUtils.setInput(Inputs.Path, inputPath);
const createTarMock = jest.spyOn(tar, "createTar"); const execMock = jest.spyOn(exec, "exec");
const cacheSize = 6 * 1024 * 1024 * 1024; //~6GB, over the 5GB limit const cacheSize = 4 * 1024 * 1024 * 1024; //~4GB, over the 2GB limit
jest.spyOn(actionUtils, "getArchiveFileSize").mockImplementationOnce(() => { jest.spyOn(actionUtils, "getArchiveFileSize").mockImplementationOnce(() => {
return cacheSize; return cacheSize;
}); });
@ -203,68 +209,30 @@ test("save with large cache outputs warning", async () => {
const archivePath = path.join("/foo/bar", "cache.tgz"); const archivePath = path.join("/foo/bar", "cache.tgz");
expect(createTarMock).toHaveBeenCalledTimes(1); const IS_WINDOWS = process.platform === "win32";
expect(createTarMock).toHaveBeenCalledWith(archivePath, cachePath); const args = IS_WINDOWS
? [
"-cz",
"--force-local",
"-f",
archivePath.replace(/\\/g, "/"),
"-C",
cachePath.replace(/\\/g, "/"),
"."
]
: ["-cz", "-f", archivePath, "-C", cachePath, "."];
expect(execMock).toHaveBeenCalledTimes(1);
expect(execMock).toHaveBeenCalledWith(`"tar"`, args);
expect(logWarningMock).toHaveBeenCalledTimes(1); expect(logWarningMock).toHaveBeenCalledTimes(1);
expect(logWarningMock).toHaveBeenCalledWith( expect(logWarningMock).toHaveBeenCalledWith(
"Cache size of ~6144 MB (6442450944 B) is over the 5GB limit, not saving cache." "Cache size of ~4 GB (4294967296 B) is over the 2GB limit, not saving cache."
); );
expect(failedMock).toHaveBeenCalledTimes(0); expect(failedMock).toHaveBeenCalledTimes(0);
}); });
test("save with reserve cache failure outputs warning", async () => {
const infoMock = jest.spyOn(core, "info");
const logWarningMock = jest.spyOn(actionUtils, "logWarning");
const failedMock = jest.spyOn(core, "setFailed");
const primaryKey = "Linux-node-bb828da54c148048dd17899ba9fda624811cfb43";
const cacheEntry: ArtifactCacheEntry = {
cacheKey: "Linux-node-",
scope: "refs/heads/master",
creationTime: "2019-11-13T19:18:02+00:00",
archiveLocation: "www.actionscache.test/download"
};
jest.spyOn(core, "getState")
// Cache Entry State
.mockImplementationOnce(() => {
return JSON.stringify(cacheEntry);
})
// Cache Key State
.mockImplementationOnce(() => {
return primaryKey;
});
const inputPath = "node_modules";
testUtils.setInput(Inputs.Path, inputPath);
const reserveCacheMock = jest
.spyOn(cacheHttpClient, "reserveCache")
.mockImplementationOnce(() => {
return Promise.resolve(-1);
});
const createTarMock = jest.spyOn(tar, "createTar");
const saveCacheMock = jest.spyOn(cacheHttpClient, "saveCache");
await run();
expect(reserveCacheMock).toHaveBeenCalledTimes(1);
expect(reserveCacheMock).toHaveBeenCalledWith(primaryKey);
expect(infoMock).toHaveBeenCalledWith(
`Unable to reserve cache with key ${primaryKey}, another job may be creating this cache.`
);
expect(createTarMock).toHaveBeenCalledTimes(0);
expect(saveCacheMock).toHaveBeenCalledTimes(0);
expect(logWarningMock).toHaveBeenCalledTimes(0);
expect(failedMock).toHaveBeenCalledTimes(0);
});
test("save with server error outputs warning", async () => { test("save with server error outputs warning", async () => {
const logWarningMock = jest.spyOn(actionUtils, "logWarning"); const logWarningMock = jest.spyOn(actionUtils, "logWarning");
const failedMock = jest.spyOn(core, "setFailed"); const failedMock = jest.spyOn(core, "setFailed");
@ -292,13 +260,11 @@ test("save with server error outputs warning", async () => {
testUtils.setInput(Inputs.Path, inputPath); testUtils.setInput(Inputs.Path, inputPath);
const cacheId = 4; const cacheId = 4;
const reserveCacheMock = jest const reserveCacheMock = jest.spyOn(cacheHttpClient, "reserveCache").mockImplementationOnce(() => {
.spyOn(cacheHttpClient, "reserveCache")
.mockImplementationOnce(() => {
return Promise.resolve(cacheId); return Promise.resolve(cacheId);
}); });
const createTarMock = jest.spyOn(tar, "createTar"); const execMock = jest.spyOn(exec, "exec");
const saveCacheMock = jest const saveCacheMock = jest
.spyOn(cacheHttpClient, "saveCache") .spyOn(cacheHttpClient, "saveCache")
@ -313,8 +279,21 @@ test("save with server error outputs warning", async () => {
const archivePath = path.join("/foo/bar", "cache.tgz"); const archivePath = path.join("/foo/bar", "cache.tgz");
expect(createTarMock).toHaveBeenCalledTimes(1); const IS_WINDOWS = process.platform === "win32";
expect(createTarMock).toHaveBeenCalledWith(archivePath, cachePath); const args = IS_WINDOWS
? [
"-cz",
"--force-local",
"-f",
archivePath.replace(/\\/g, "/"),
"-C",
cachePath.replace(/\\/g, "/"),
"."
]
: ["-cz", "-f", archivePath, "-C", cachePath, "."];
expect(execMock).toHaveBeenCalledTimes(1);
expect(execMock).toHaveBeenCalledWith(`"tar"`, args);
expect(saveCacheMock).toHaveBeenCalledTimes(1); expect(saveCacheMock).toHaveBeenCalledTimes(1);
expect(saveCacheMock).toHaveBeenCalledWith(cacheId, archivePath); expect(saveCacheMock).toHaveBeenCalledWith(cacheId, archivePath);
@ -351,13 +330,11 @@ test("save with valid inputs uploads a cache", async () => {
testUtils.setInput(Inputs.Path, inputPath); testUtils.setInput(Inputs.Path, inputPath);
const cacheId = 4; const cacheId = 4;
const reserveCacheMock = jest const reserveCacheMock = jest.spyOn(cacheHttpClient, "reserveCache").mockImplementationOnce(() => {
.spyOn(cacheHttpClient, "reserveCache")
.mockImplementationOnce(() => {
return Promise.resolve(cacheId); return Promise.resolve(cacheId);
}); });
const createTarMock = jest.spyOn(tar, "createTar"); const execMock = jest.spyOn(exec, "exec");
const saveCacheMock = jest.spyOn(cacheHttpClient, "saveCache"); const saveCacheMock = jest.spyOn(cacheHttpClient, "saveCache");
@ -368,8 +345,21 @@ test("save with valid inputs uploads a cache", async () => {
const archivePath = path.join("/foo/bar", "cache.tgz"); const archivePath = path.join("/foo/bar", "cache.tgz");
expect(createTarMock).toHaveBeenCalledTimes(1); const IS_WINDOWS = process.platform === "win32";
expect(createTarMock).toHaveBeenCalledWith(archivePath, cachePath); const args = IS_WINDOWS
? [
"-cz",
"--force-local",
"-f",
archivePath.replace(/\\/g, "/"),
"-C",
cachePath.replace(/\\/g, "/"),
"."
]
: ["-cz", "-f", archivePath, "-C", cachePath, "."];
expect(execMock).toHaveBeenCalledTimes(1);
expect(execMock).toHaveBeenCalledWith(`"tar"`, args);
expect(saveCacheMock).toHaveBeenCalledTimes(1); expect(saveCacheMock).toHaveBeenCalledTimes(1);
expect(saveCacheMock).toHaveBeenCalledWith(cacheId, archivePath); expect(saveCacheMock).toHaveBeenCalledWith(cacheId, archivePath);

View File

@ -1,58 +0,0 @@
import * as exec from "@actions/exec";
import * as io from "@actions/io";
import * as tar from "../src/tar";
jest.mock("@actions/exec");
jest.mock("@actions/io");
beforeAll(() => {
jest.spyOn(io, "which").mockImplementation(tool => {
return Promise.resolve(tool);
});
});
test("extract tar", async () => {
const mkdirMock = jest.spyOn(io, "mkdirP");
const execMock = jest.spyOn(exec, "exec");
const archivePath = "cache.tar";
const targetDirectory = "~/.npm/cache";
await tar.extractTar(archivePath, targetDirectory);
expect(mkdirMock).toHaveBeenCalledWith(targetDirectory);
const IS_WINDOWS = process.platform === "win32";
const tarPath = IS_WINDOWS
? `${process.env["windir"]}\\System32\\tar.exe`
: "tar";
expect(execMock).toHaveBeenCalledTimes(1);
expect(execMock).toHaveBeenCalledWith(`"${tarPath}"`, [
"-xz",
"-f",
archivePath,
"-C",
targetDirectory
]);
});
test("create tar", async () => {
const execMock = jest.spyOn(exec, "exec");
const archivePath = "cache.tar";
const sourceDirectory = "~/.npm/cache";
await tar.createTar(archivePath, sourceDirectory);
const IS_WINDOWS = process.platform === "win32";
const tarPath = IS_WINDOWS
? `${process.env["windir"]}\\System32\\tar.exe`
: "tar";
expect(execMock).toHaveBeenCalledTimes(1);
expect(execMock).toHaveBeenCalledWith(`"${tarPath}"`, [
"-cz",
"-f",
archivePath,
"-C",
sourceDirectory,
"."
]);
});

View File

@ -1,5 +1,5 @@
name: 'Cache' name: 'Cache'
description: 'Cache artifacts like dependencies and build outputs to improve workflow execution time' description: 'Cache dependencies and build outputs to improve workflow execution time'
author: 'GitHub' author: 'GitHub'
inputs: inputs:
path: path:

3624
dist/restore/index.js vendored

File diff suppressed because it is too large Load Diff

3625
dist/save/index.js vendored

File diff suppressed because it is too large Load Diff

View File

@ -1,24 +1,20 @@
# Examples # Examples
- [C# - NuGet](#c---nuget) - [C# - Nuget](#c---nuget)
- [Elixir - Mix](#elixir---mix) - [Elixir - Mix](#elixir---mix)
- [Go - Modules](#go---modules) - [Go - Modules](#go---modules)
- [Haskell - Cabal](#haskell---cabal)
- [Java - Gradle](#java---gradle) - [Java - Gradle](#java---gradle)
- [Java - Maven](#java---maven) - [Java - Maven](#java---maven)
- [Node - npm](#node---npm) - [Node - npm](#node---npm)
- [Node - Yarn](#node---yarn) - [Node - Yarn](#node---yarn)
- [PHP - Composer](#php---composer) - [PHP - Composer](#php---composer)
- [Python - pip](#python---pip) - [Python - pip](#python---pip)
- [R - renv](#r---renv) - [Ruby - Gem](#ruby---gem)
- [Ruby - Bundler](#ruby---bundler)
- [Rust - Cargo](#rust---cargo) - [Rust - Cargo](#rust---cargo)
- [Scala - SBT](#scala---sbt)
- [Swift, Objective-C - Carthage](#swift-objective-c---carthage) - [Swift, Objective-C - Carthage](#swift-objective-c---carthage)
- [Swift, Objective-C - CocoaPods](#swift-objective-c---cocoapods) - [Swift, Objective-C - CocoaPods](#swift-objective-c---cocoapods)
- [Swift - Swift Package Manager](#swift---swift-package-manager)
## C# - NuGet ## C# - Nuget
Using [NuGet lock files](https://docs.microsoft.com/nuget/consume-packages/package-references-in-project-files#locking-dependencies): Using [NuGet lock files](https://docs.microsoft.com/nuget/consume-packages/package-references-in-project-files#locking-dependencies):
```yaml ```yaml
@ -30,21 +26,6 @@ Using [NuGet lock files](https://docs.microsoft.com/nuget/consume-packages/packa
${{ runner.os }}-nuget- ${{ runner.os }}-nuget-
``` ```
Depending on the environment, huge packages might be pre-installed in the global cache folder.
If you do not want to include them, consider to move the cache folder like below.
>Note: This workflow does not work for projects that require files to be placed in user profile package folder
```yaml
env:
NUGET_PACKAGES: ${{ github.workspace }}/.nuget/packages
steps:
- uses: actions/cache@v1
with:
path: ${{ github.workspace }}/.nuget/packages
key: ${{ runner.os }}-nuget-${{ hashFiles('**/packages.lock.json') }}
restore-keys: |
${{ runner.os }}-nuget-
```
## Elixir - Mix ## Elixir - Mix
```yaml ```yaml
- uses: actions/cache@v1 - uses: actions/cache@v1
@ -66,28 +47,6 @@ steps:
${{ runner.os }}-go- ${{ runner.os }}-go-
``` ```
## Haskell - Cabal
We cache the elements of the Cabal store separately, as the entirety of `~/.cabal` can grow very large for projects with many dependencies.
```yaml
- uses: actions/cache@v1
name: Cache ~/.cabal/packages
with:
path: ~/.cabal/packages
key: ${{ runner.os }}-${{ matrix.ghc }}-cabal-packages
- uses: actions/cache@v1
name: Cache ~/.cabal/store
with:
path: ~/.cabal/store
key: ${{ runner.os }}-${{ matrix.ghc }}-cabal-store
- uses: actions/cache@v1
name: Cache dist-newstyle
with:
path: dist-newstyle
key: ${{ runner.os }}-${{ matrix.ghc }}-dist-newstyle
```
## Java - Gradle ## Java - Gradle
```yaml ```yaml
@ -250,64 +209,15 @@ Replace `~/.cache/pip` with the correct `path` if not using Ubuntu.
${{ runner.os }}-pip- ${{ runner.os }}-pip-
``` ```
## R - renv ## Ruby - Gem
For renv, the cache directory will vary by OS. Look at https://rstudio.github.io/renv/articles/renv.html#cache
Locations:
- Ubuntu: `~/.local/share/renv`
- macOS: `~/Library/Application Support/renv`
- Windows: `%LOCALAPPDATA%/renv`
### Simple example
```yaml
- uses: actions/cache@v1
with:
path: ~/.local/share/renv
key: ${{ runner.os }}-renv-${{ hashFiles('**/renv.lock') }}
restore-keys: |
${{ runner.os }}-renv-
```
Replace `~/.local/share/renv` with the correct `path` if not using Ubuntu.
### Multiple OS's in a workflow
```yaml
- uses: actions/cache@v1
if: startsWith(runner.os, 'Linux')
with:
path: ~/.local/share/renv
key: ${{ runner.os }}-renv-${{ hashFiles('**/renv.lock') }}
restore-keys: |
${{ runner.os }}-renv-
- uses: actions/cache@v1
if: startsWith(runner.os, 'macOS')
with:
path: ~/Library/Application Support/renv
key: ${{ runner.os }}-renv-${{ hashFiles('**/renv.lock') }}
restore-keys: |
${{ runner.os }}-renv-
- uses: actions/cache@v1
if: startsWith(runner.os, 'Windows')
with:
path: ~\AppData\Local\renv
key: ${{ runner.os }}-renv-${{ hashFiles('**/renv.lock') }}
restore-keys: |
${{ runner.os }}-renv-
```
## Ruby - Bundler
```yaml ```yaml
- uses: actions/cache@v1 - uses: actions/cache@v1
with: with:
path: vendor/bundle path: vendor/bundle
key: ${{ runner.os }}-gems-${{ hashFiles('**/Gemfile.lock') }} key: ${{ runner.os }}-gem-${{ hashFiles('**/Gemfile.lock') }}
restore-keys: | restore-keys: |
${{ runner.os }}-gems- ${{ runner.os }}-gem-
``` ```
When dependencies are installed later in the workflow, we must specify the same path for the bundler. When dependencies are installed later in the workflow, we must specify the same path for the bundler.
@ -338,21 +248,6 @@ When dependencies are installed later in the workflow, we must specify the same
key: ${{ runner.os }}-cargo-build-target-${{ hashFiles('**/Cargo.lock') }} key: ${{ runner.os }}-cargo-build-target-${{ hashFiles('**/Cargo.lock') }}
``` ```
## Scala - SBT
```yaml
- name: Cache SBT ivy cache
uses: actions/cache@v1
with:
path: ~/.ivy2/cache
key: ${{ runner.os }}-sbt-ivy-cache-${{ hashFiles('**/build.sbt') }}
- name: Cache SBT
uses: actions/cache@v1
with:
path: ~/.sbt
key: ${{ runner.os }}-sbt-${{ hashFiles('**/build.sbt') }}
```
## Swift, Objective-C - Carthage ## Swift, Objective-C - Carthage
```yaml ```yaml
@ -374,14 +269,3 @@ When dependencies are installed later in the workflow, we must specify the same
restore-keys: | restore-keys: |
${{ runner.os }}-pods- ${{ runner.os }}-pods-
``` ```
## Swift - Swift Package Manager
```yaml
- uses: actions/cache@v1
with:
path: .build
key: ${{ runner.os }}-spm-${{ hashFiles('**/Package.resolved') }}
restore-keys: |
${{ runner.os }}-spm-
```

42
package-lock.json generated
View File

@ -1,6 +1,6 @@
{ {
"name": "cache", "name": "cache",
"version": "1.1.2", "version": "1.0.3",
"lockfileVersion": 1, "lockfileVersion": 1,
"requires": true, "requires": true,
"dependencies": { "dependencies": {
@ -14,14 +14,6 @@
"resolved": "https://registry.npmjs.org/@actions/exec/-/exec-1.0.1.tgz", "resolved": "https://registry.npmjs.org/@actions/exec/-/exec-1.0.1.tgz",
"integrity": "sha512-nvFkxwiicvpzNiCBF4wFBDfnBvi7xp/as7LE1hBxBxKG2L29+gkIPBiLKMVORL+Hg3JNf07AKRfl0V5djoypjQ==" "integrity": "sha512-nvFkxwiicvpzNiCBF4wFBDfnBvi7xp/as7LE1hBxBxKG2L29+gkIPBiLKMVORL+Hg3JNf07AKRfl0V5djoypjQ=="
}, },
"@actions/http-client": {
"version": "1.0.6",
"resolved": "https://registry.npmjs.org/@actions/http-client/-/http-client-1.0.6.tgz",
"integrity": "sha512-LGmio4w98UyGX33b/W6V6Nx/sQHRXZ859YlMkn36wPsXPB82u8xTVlA/Dq2DXrm6lEq9RVmisRJa1c+HETAIJA==",
"requires": {
"tunnel": "0.0.6"
}
},
"@actions/io": { "@actions/io": {
"version": "1.0.1", "version": "1.0.1",
"resolved": "https://registry.npmjs.org/@actions/io/-/io-1.0.1.tgz", "resolved": "https://registry.npmjs.org/@actions/io/-/io-1.0.1.tgz",
@ -2862,9 +2854,9 @@
"dev": true "dev": true
}, },
"handlebars": { "handlebars": {
"version": "4.5.3", "version": "4.5.1",
"resolved": "https://registry.npmjs.org/handlebars/-/handlebars-4.5.3.tgz", "resolved": "https://registry.npmjs.org/handlebars/-/handlebars-4.5.1.tgz",
"integrity": "sha512-3yPecJoJHK/4c6aZhSvxOyG4vJKDshV36VHp0iVCDVh7o9w2vwi3NSnL2MMPj3YdduqaBcu7cGbggJQM0br9xA==", "integrity": "sha512-C29UoFzHe9yM61lOsIlCE5/mQVGrnIOrOq7maQl76L7tYPCgC1og0Ajt6uWnX4ZTxBPnjw+CUvawphwCfJgUnA==",
"dev": true, "dev": true,
"requires": { "requires": {
"neo-async": "^2.6.0", "neo-async": "^2.6.0",
@ -5941,9 +5933,9 @@
} }
}, },
"tunnel": { "tunnel": {
"version": "0.0.6", "version": "0.0.4",
"resolved": "https://registry.npmjs.org/tunnel/-/tunnel-0.0.6.tgz", "resolved": "https://registry.npmjs.org/tunnel/-/tunnel-0.0.4.tgz",
"integrity": "sha512-1h/Lnq9yajKY2PEbBadPXj3VxsDDu844OnaAo52UVmIzIvwwtBPIuNvkjuzBlTWpfJyUbG3ez0KSBibQkj4ojg==" "integrity": "sha1-LTeFoVjBdMmhbcLARuxfxfF0IhM="
}, },
"tunnel-agent": { "tunnel-agent": {
"version": "0.6.0", "version": "0.6.0",
@ -5981,6 +5973,15 @@
"integrity": "sha512-DWkS49EQKVX//Tbupb9TFa19c7+MK1XmzkrZUR8TAktmE/DizXoaoJV6TZ/tSIPXipqNiRI6CyAe7x69Jb6RSw==", "integrity": "sha512-DWkS49EQKVX//Tbupb9TFa19c7+MK1XmzkrZUR8TAktmE/DizXoaoJV6TZ/tSIPXipqNiRI6CyAe7x69Jb6RSw==",
"dev": true "dev": true
}, },
"typed-rest-client": {
"version": "1.5.0",
"resolved": "https://registry.npmjs.org/typed-rest-client/-/typed-rest-client-1.5.0.tgz",
"integrity": "sha512-DVZRlmsfnTjp6ZJaatcdyvvwYwbWvR4YDNFDqb+qdTxpvaVP99YCpBkA8rxsLtAPjBVoDe4fNsnMIdZTiPuKWg==",
"requires": {
"tunnel": "0.0.4",
"underscore": "1.8.3"
}
},
"typescript": { "typescript": {
"version": "3.7.3", "version": "3.7.3",
"resolved": "https://registry.npmjs.org/typescript/-/typescript-3.7.3.tgz", "resolved": "https://registry.npmjs.org/typescript/-/typescript-3.7.3.tgz",
@ -5988,9 +5989,9 @@
"dev": true "dev": true
}, },
"uglify-js": { "uglify-js": {
"version": "3.7.3", "version": "3.6.7",
"resolved": "https://registry.npmjs.org/uglify-js/-/uglify-js-3.7.3.tgz", "resolved": "https://registry.npmjs.org/uglify-js/-/uglify-js-3.6.7.tgz",
"integrity": "sha512-7tINm46/3puUA4hCkKYo4Xdts+JDaVC9ZPRcG8Xw9R4nhO/gZgUM3TENq8IF4Vatk8qCig4MzP/c8G4u2BkVQg==", "integrity": "sha512-4sXQDzmdnoXiO+xvmTzQsfIiwrjUCSA95rSP4SEd8tDb51W2TiDOlL76Hl+Kw0Ie42PSItCW8/t6pBNCF2R48A==",
"dev": true, "dev": true,
"optional": true, "optional": true,
"requires": { "requires": {
@ -5998,6 +5999,11 @@
"source-map": "~0.6.1" "source-map": "~0.6.1"
} }
}, },
"underscore": {
"version": "1.8.3",
"resolved": "https://registry.npmjs.org/underscore/-/underscore-1.8.3.tgz",
"integrity": "sha1-Tz+1OxBuYJf8+ctBCfKl6b36UCI="
},
"union-value": { "union-value": {
"version": "1.0.1", "version": "1.0.1",
"resolved": "https://registry.npmjs.org/union-value/-/union-value-1.0.1.tgz", "resolved": "https://registry.npmjs.org/union-value/-/union-value-1.0.1.tgz",

View File

@ -1,6 +1,6 @@
{ {
"name": "cache", "name": "cache",
"version": "1.1.2", "version": "1.1.0",
"private": true, "private": true,
"description": "Cache dependencies and build outputs", "description": "Cache dependencies and build outputs",
"main": "dist/restore/index.js", "main": "dist/restore/index.js",
@ -26,8 +26,8 @@
"dependencies": { "dependencies": {
"@actions/core": "^1.2.0", "@actions/core": "^1.2.0",
"@actions/exec": "^1.0.1", "@actions/exec": "^1.0.1",
"@actions/http-client": "^1.0.6",
"@actions/io": "^1.0.1", "@actions/io": "^1.0.1",
"typed-rest-client": "^1.5.0",
"uuid": "^3.3.3" "uuid": "^3.3.3"
}, },
"devDependencies": { "devDependencies": {

View File

@ -1,40 +1,25 @@
import * as core from "@actions/core"; import * as core from "@actions/core";
import * as fs from "fs"; import * as fs from "fs";
import { BearerCredentialHandler } from "@actions/http-client/auth"; import { BearerCredentialHandler } from "typed-rest-client/Handlers";
import { HttpClient, HttpCodes } from "@actions/http-client"; import { HttpClient } from "typed-rest-client/HttpClient";
import { IHttpClientResponse } from "typed-rest-client/Interfaces";
import { import {
IHttpClientResponse,
IRequestOptions, IRequestOptions,
ITypedResponse RestClient,
} from "@actions/http-client/interfaces"; IRestResponse
} from "typed-rest-client/RestClient";
import { import {
ArtifactCacheEntry, ArtifactCacheEntry,
CommitCacheRequest, CommitCacheRequest,
ReserveCacheRequest, ReserveCacheRequest,
ReserveCacheResponse ReserverCacheResponse
} from "./contracts"; } from "./contracts";
import * as utils from "./utils/actionUtils"; import * as utils from "./utils/actionUtils";
function isSuccessStatusCode(statusCode?: number): boolean { function isSuccessStatusCode(statusCode: number): boolean {
if (!statusCode) {
return false;
}
return statusCode >= 200 && statusCode < 300; return statusCode >= 200 && statusCode < 300;
} }
function getCacheApiUrl(): string {
function isRetryableStatusCode(statusCode?: number): boolean {
if (!statusCode) {
return false;
}
const retryableStatusCodes = [
HttpCodes.BadGateway,
HttpCodes.ServiceUnavailable,
HttpCodes.GatewayTimeout
];
return retryableStatusCodes.includes(statusCode);
}
function getCacheApiUrl(resource: string): string {
// Ideally we just use ACTIONS_CACHE_URL // Ideally we just use ACTIONS_CACHE_URL
const baseUrl: string = ( const baseUrl: string = (
process.env["ACTIONS_CACHE_URL"] || process.env["ACTIONS_CACHE_URL"] ||
@ -47,9 +32,8 @@ function getCacheApiUrl(resource: string): string {
); );
} }
const url = `${baseUrl}_apis/artifactcache/${resource}`; core.debug(`Cache Url: ${baseUrl}`);
core.debug(`Resource Url: ${url}`); return `${baseUrl}_apis/artifactcache/`;
return url;
} }
function createAcceptHeader(type: string, apiVersion: string): string { function createAcceptHeader(type: string, apiVersion: string): string {
@ -58,33 +42,30 @@ function createAcceptHeader(type: string, apiVersion: string): string {
function getRequestOptions(): IRequestOptions { function getRequestOptions(): IRequestOptions {
const requestOptions: IRequestOptions = { const requestOptions: IRequestOptions = {
headers: { acceptHeader: createAcceptHeader("application/json", "6.0-preview.1")
Accept: createAcceptHeader("application/json", "6.0-preview.1")
}
}; };
return requestOptions; return requestOptions;
} }
function createHttpClient(): HttpClient { function createRestClient(): RestClient {
const token = process.env["ACTIONS_RUNTIME_TOKEN"] || ""; const token = process.env["ACTIONS_RUNTIME_TOKEN"] || "";
const bearerCredentialHandler = new BearerCredentialHandler(token); const bearerCredentialHandler = new BearerCredentialHandler(token);
return new HttpClient( return new RestClient("actions/cache", getCacheApiUrl(), [
"actions/cache", bearerCredentialHandler
[bearerCredentialHandler], ]);
getRequestOptions()
);
} }
export async function getCacheEntry( export async function getCacheEntry(
keys: string[] keys: string[]
): Promise<ArtifactCacheEntry | null> { ): Promise<ArtifactCacheEntry | null> {
const httpClient = createHttpClient(); const restClient = createRestClient();
const resource = `cache?keys=${encodeURIComponent(keys.join(","))}`; const resource = `cache?keys=${encodeURIComponent(keys.join(","))}`;
const response = await httpClient.getJson<ArtifactCacheEntry>( const response = await restClient.get<ArtifactCacheEntry>(
getCacheApiUrl(resource) resource,
getRequestOptions()
); );
if (response.statusCode === 204) { if (response.statusCode === 204) {
return null; return null;
@ -92,7 +73,6 @@ export async function getCacheEntry(
if (!isSuccessStatusCode(response.statusCode)) { if (!isSuccessStatusCode(response.statusCode)) {
throw new Error(`Cache service responded with ${response.statusCode}`); throw new Error(`Cache service responded with ${response.statusCode}`);
} }
const cacheResult = response.result; const cacheResult = response.result;
const cacheDownloadUrl = cacheResult?.archiveLocation; const cacheDownloadUrl = cacheResult?.archiveLocation;
if (!cacheDownloadUrl) { if (!cacheDownloadUrl) {
@ -128,15 +108,17 @@ export async function downloadCache(
// Reserve Cache // Reserve Cache
export async function reserveCache(key: string): Promise<number> { export async function reserveCache(key: string): Promise<number> {
const httpClient = createHttpClient(); const restClient = createRestClient();
const reserveCacheRequest: ReserveCacheRequest = { const reserveCacheRequest: ReserveCacheRequest = {
key key
}; };
const response = await httpClient.postJson<ReserveCacheResponse>( const response = await restClient.create<ReserverCacheResponse>(
getCacheApiUrl("caches"), "caches",
reserveCacheRequest reserveCacheRequest,
getRequestOptions()
); );
return response?.result?.cacheId ?? -1; return response?.result?.cacheId ?? -1;
} }
@ -150,12 +132,12 @@ function getContentRange(start: number, end: number): string {
} }
async function uploadChunk( async function uploadChunk(
httpClient: HttpClient, restClient: RestClient,
resourceUrl: string, resourceUrl: string,
data: NodeJS.ReadableStream, data: NodeJS.ReadableStream,
start: number, start: number,
end: number end: number
): Promise<void> { ): Promise<IRestResponse<void>> {
core.debug( core.debug(
`Uploading chunk of size ${end - `Uploading chunk of size ${end -
start + start +
@ -164,75 +146,45 @@ async function uploadChunk(
end end
)}` )}`
); );
const additionalHeaders = { const requestOptions = getRequestOptions();
requestOptions.additionalHeaders = {
"Content-Type": "application/octet-stream", "Content-Type": "application/octet-stream",
"Content-Range": getContentRange(start, end) "Content-Range": getContentRange(start, end)
}; };
const uploadChunkRequest = async (): Promise<IHttpClientResponse> => { return await restClient.uploadStream<void>(
return await httpClient.sendStream(
"PATCH", "PATCH",
resourceUrl, resourceUrl,
data, data,
additionalHeaders requestOptions
); );
};
const response = await uploadChunkRequest();
if (isSuccessStatusCode(response.message.statusCode)) {
return;
}
if (isRetryableStatusCode(response.message.statusCode)) {
core.debug(
`Received ${response.message.statusCode}, retrying chunk at offset ${start}.`
);
const retryResponse = await uploadChunkRequest();
if (isSuccessStatusCode(retryResponse.message.statusCode)) {
return;
}
}
throw new Error(
`Cache service responded with ${response.message.statusCode} during chunk upload.`
);
}
function parseEnvNumber(key: string): number | undefined {
const value = Number(process.env[key]);
if (Number.isNaN(value) || value < 0) {
return undefined;
}
return value;
} }
async function uploadFile( async function uploadFile(
httpClient: HttpClient, restClient: RestClient,
cacheId: number, cacheId: number,
archivePath: string archivePath: string
): Promise<void> { ): Promise<void> {
// Upload Chunks // Upload Chunks
const fileSize = fs.statSync(archivePath).size; const fileSize = fs.statSync(archivePath).size;
const resourceUrl = getCacheApiUrl(`caches/${cacheId.toString()}`); const resourceUrl = getCacheApiUrl() + "caches/" + cacheId.toString();
const responses: IRestResponse<void>[] = [];
const fd = fs.openSync(archivePath, "r"); const fd = fs.openSync(archivePath, "r");
const concurrency = parseEnvNumber("CACHE_UPLOAD_CONCURRENCY") ?? 4; // # of HTTP requests in parallel const concurrency = 4; // # of HTTP requests in parallel
const MAX_CHUNK_SIZE = const MAX_CHUNK_SIZE = 32000000; // 32 MB Chunks
parseEnvNumber("CACHE_UPLOAD_CHUNK_SIZE") ?? 32 * 1024 * 1024; // 32 MB Chunks
core.debug(`Concurrency: ${concurrency} and Chunk Size: ${MAX_CHUNK_SIZE}`); core.debug(`Concurrency: ${concurrency} and Chunk Size: ${MAX_CHUNK_SIZE}`);
const parallelUploads = [...new Array(concurrency).keys()]; const parallelUploads = [...new Array(concurrency).keys()];
core.debug("Awaiting all uploads"); core.debug("Awaiting all uploads");
let offset = 0; let offset = 0;
try {
await Promise.all( await Promise.all(
parallelUploads.map(async () => { parallelUploads.map(async () => {
while (offset < fileSize) { while (offset < fileSize) {
const chunkSize = Math.min( const chunkSize =
fileSize - offset, offset + MAX_CHUNK_SIZE > fileSize
MAX_CHUNK_SIZE ? fileSize - offset
); : MAX_CHUNK_SIZE;
const start = offset; const start = offset;
const end = offset + chunkSize - 1; const end = offset + chunkSize - 1;
offset += MAX_CHUNK_SIZE; offset += MAX_CHUNK_SIZE;
@ -242,32 +194,44 @@ async function uploadFile(
end, end,
autoClose: false autoClose: false
}); });
responses.push(
await uploadChunk( await uploadChunk(
httpClient, restClient,
resourceUrl, resourceUrl,
chunk, chunk,
start, start,
end end
)
); );
} }
}) })
); );
} finally {
fs.closeSync(fd); fs.closeSync(fd);
const failedResponse = responses.find(
x => !isSuccessStatusCode(x.statusCode)
);
if (failedResponse) {
throw new Error(
`Cache service responded with ${failedResponse.statusCode} during chunk upload.`
);
} }
return; return;
} }
async function commitCache( async function commitCache(
httpClient: HttpClient, restClient: RestClient,
cacheId: number, cacheId: number,
filesize: number filesize: number
): Promise<ITypedResponse<null>> { ): Promise<IRestResponse<void>> {
const requestOptions = getRequestOptions();
const commitCacheRequest: CommitCacheRequest = { size: filesize }; const commitCacheRequest: CommitCacheRequest = { size: filesize };
return await httpClient.postJson<null>( return await restClient.create(
getCacheApiUrl(`caches/${cacheId.toString()}`), `caches/${cacheId.toString()}`,
commitCacheRequest commitCacheRequest,
requestOptions
); );
} }
@ -275,16 +239,16 @@ export async function saveCache(
cacheId: number, cacheId: number,
archivePath: string archivePath: string
): Promise<void> { ): Promise<void> {
const httpClient = createHttpClient(); const restClient = createRestClient();
core.debug("Upload cache"); core.debug("Upload cache");
await uploadFile(httpClient, cacheId, archivePath); await uploadFile(restClient, cacheId, archivePath);
// Commit Cache // Commit Cache
core.debug("Commiting cache"); core.debug("Commiting cache");
const cacheSize = utils.getArchiveFileSize(archivePath); const cacheSize = utils.getArchiveFileSize(archivePath);
const commitCacheResponse = await commitCache( const commitCacheResponse = await commitCache(
httpClient, restClient,
cacheId, cacheId,
cacheSize cacheSize
); );

2
src/contracts.d.ts vendored
View File

@ -14,6 +14,6 @@ export interface ReserveCacheRequest {
version?: string; version?: string;
} }
export interface ReserveCacheResponse { export interface ReserverCacheResponse {
cacheId: number; cacheId: number;
} }

View File

@ -1,8 +1,9 @@
import * as core from "@actions/core"; import * as core from "@actions/core";
import { exec } from "@actions/exec";
import * as io from "@actions/io";
import * as path from "path"; import * as path from "path";
import * as cacheHttpClient from "./cacheHttpClient"; import * as cacheHttpClient from "./cacheHttpClient";
import { Events, Inputs, State } from "./constants"; import { Events, Inputs, State } from "./constants";
import { extractTar } from "./tar";
import * as utils from "./utils/actionUtils"; import * as utils from "./utils/actionUtils";
async function run(): Promise<void> { async function run(): Promise<void> {
@ -60,7 +61,7 @@ async function run(): Promise<void> {
try { try {
const cacheEntry = await cacheHttpClient.getCacheEntry(keys); const cacheEntry = await cacheHttpClient.getCacheEntry(keys);
if (!cacheEntry?.archiveLocation) { if (!cacheEntry || !cacheEntry?.archiveLocation) {
core.info( core.info(
`Cache not found for input keys: ${keys.join(", ")}.` `Cache not found for input keys: ${keys.join(", ")}.`
); );
@ -78,7 +79,7 @@ async function run(): Promise<void> {
// Download the cache from the cache entry // Download the cache from the cache entry
await cacheHttpClient.downloadCache( await cacheHttpClient.downloadCache(
cacheEntry.archiveLocation, cacheEntry?.archiveLocation,
archivePath archivePath
); );
@ -89,7 +90,27 @@ async function run(): Promise<void> {
)} MB (${archiveFileSize} B)` )} MB (${archiveFileSize} B)`
); );
await extractTar(archivePath, cachePath); // Create directory to extract tar into
await io.mkdirP(cachePath);
// http://man7.org/linux/man-pages/man1/tar.1.html
// tar [-options] <name of the tar archive> [files or directories which to add into archive]
const IS_WINDOWS = process.platform === "win32";
const args = IS_WINDOWS
? [
"-xz",
"--force-local",
"-f",
archivePath.replace(/\\/g, "/"),
"-C",
cachePath.replace(/\\/g, "/")
]
: ["-xz", "-f", archivePath, "-C", cachePath];
const tarPath = await io.which("tar", true);
core.debug(`Tar Path: ${tarPath}`);
await exec(`"${tarPath}"`, args);
const isExactKeyMatch = utils.isExactKeyMatch( const isExactKeyMatch = utils.isExactKeyMatch(
primaryKey, primaryKey,

View File

@ -1,8 +1,9 @@
import * as core from "@actions/core"; import * as core from "@actions/core";
import { exec } from "@actions/exec";
import * as io from "@actions/io";
import * as path from "path"; import * as path from "path";
import * as cacheHttpClient from "./cacheHttpClient"; import * as cacheHttpClient from "./cacheHttpClient";
import { Events, Inputs, State } from "./constants"; import { Events, Inputs, State } from "./constants";
import { createTar } from "./tar";
import * as utils from "./utils/actionUtils"; import * as utils from "./utils/actionUtils";
async function run(): Promise<void> { async function run(): Promise<void> {
@ -36,7 +37,7 @@ async function run(): Promise<void> {
core.debug("Reserving Cache"); core.debug("Reserving Cache");
const cacheId = await cacheHttpClient.reserveCache(primaryKey); const cacheId = await cacheHttpClient.reserveCache(primaryKey);
if (cacheId == -1) { if (cacheId < 0) {
core.info( core.info(
`Unable to reserve cache with key ${primaryKey}, another job may be creating this cache.` `Unable to reserve cache with key ${primaryKey}, another job may be creating this cache.`
); );
@ -54,21 +55,38 @@ async function run(): Promise<void> {
); );
core.debug(`Archive Path: ${archivePath}`); core.debug(`Archive Path: ${archivePath}`);
await createTar(archivePath, cachePath); // http://man7.org/linux/man-pages/man1/tar.1.html
// tar [-options] <name of the tar archive> [files or directories which to add into archive]
const IS_WINDOWS = process.platform === "win32";
const args = IS_WINDOWS
? [
"-cz",
"--force-local",
"-f",
archivePath.replace(/\\/g, "/"),
"-C",
cachePath.replace(/\\/g, "/"),
"."
]
: ["-cz", "-f", archivePath, "-C", cachePath, "."];
const fileSizeLimit = 5 * 1024 * 1024 * 1024; // 5GB per repo limit const tarPath = await io.which("tar", true);
core.debug(`Tar Path: ${tarPath}`);
await exec(`"${tarPath}"`, args);
const fileSizeLimit = 2 * 1024 * 1024 * 1024; // 2GB per repo limit
const archiveFileSize = utils.getArchiveFileSize(archivePath); const archiveFileSize = utils.getArchiveFileSize(archivePath);
core.debug(`File Size: ${archiveFileSize}`); core.debug(`File Size: ${archiveFileSize}`);
if (archiveFileSize > fileSizeLimit) { if (archiveFileSize > fileSizeLimit) {
utils.logWarning( utils.logWarning(
`Cache size of ~${Math.round( `Cache size of ~${Math.round(
archiveFileSize / (1024 * 1024) archiveFileSize / (1024 * 1024 * 1024)
)} MB (${archiveFileSize} B) is over the 5GB limit, not saving cache.` )} GB (${archiveFileSize} B) is over the 2GB limit, not saving cache.`
); );
return; return;
} }
core.debug(`Saving Cache (ID: ${cacheId})`); core.debug("Saving Cache");
await cacheHttpClient.saveCache(cacheId, archivePath); await cacheHttpClient.saveCache(cacheId, archivePath);
} catch (error) { } catch (error) {
utils.logWarning(error.message); utils.logWarning(error.message);

View File

@ -1,47 +0,0 @@
import { exec } from "@actions/exec";
import * as io from "@actions/io";
import { existsSync } from "fs";
async function getTarPath(): Promise<string> {
// Explicitly use BSD Tar on Windows
const IS_WINDOWS = process.platform === "win32";
if (IS_WINDOWS) {
const systemTar = `${process.env["windir"]}\\System32\\tar.exe`;
if (existsSync(systemTar)) {
return systemTar;
}
}
return await io.which("tar", true);
}
async function execTar(args: string[]): Promise<void> {
try {
await exec(`"${await getTarPath()}"`, args);
} catch (error) {
const IS_WINDOWS = process.platform === "win32";
if (IS_WINDOWS) {
throw new Error(
`Tar failed with error: ${error?.message}. Ensure BSD tar is installed and on the PATH.`
);
}
throw new Error(`Tar failed with error: ${error?.message}`);
}
}
export async function extractTar(
archivePath: string,
targetDirectory: string
): Promise<void> {
// Create directory to extract tar into
await io.mkdirP(targetDirectory);
const args = ["-xz", "-f", archivePath, "-C", targetDirectory];
await execTar(args);
}
export async function createTar(
archivePath: string,
sourceDirectory: string
): Promise<void> {
const args = ["-cz", "-f", archivePath, "-C", sourceDirectory, "."];
await execTar(args);
}