diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 612b38c..29d8e5c 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -563,3 +563,22 @@ jobs:
         uses: docker/build-push-action@master
         with:
           context: .
+
+  cacheBinary:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        cache:
+          - true
+          - false
+    steps:
+      -
+        name: Checkout
+        uses: actions/checkout@v4
+      -
+        name: Set up Docker Buildx
+        uses: ./
+        with:
+          version: v0.11.2
+          cache-binary: ${{ matrix.cache }}
diff --git a/README.md b/README.md
index aaae57c..e77f938 100644
--- a/README.md
+++ b/README.md
@@ -98,6 +98,7 @@ The following inputs can be used as `step.with` keys:
 | `config`¹         | String   |                    | [BuildKit config file](https://docs.docker.com/engine/reference/commandline/buildx_create/#config)                                                                           |
 | `config-inline`¹  | String   |                    | Same as `config` but inline                                                                                                                                                  |
 | `append`          | YAML     |                    | [Append additional nodes](docs/advanced/append-nodes.md) to the builder                                                                                                      |
+| `cache-binary`    | Bool     | `true`             | Cache buildx binary to GitHub Actions cache backend                                                                                                                          |
 | `cleanup`         | Bool     | `true`             | Cleanup temp files and remove builder at the end of a job                                                                                                                    |
 
 > * ¹ `config` and `config-inline` are mutually exclusive
diff --git a/__tests__/context.test.ts b/__tests__/context.test.ts
index e7b8564..1342fd2 100644
--- a/__tests__/context.test.ts
+++ b/__tests__/context.test.ts
@@ -32,6 +32,7 @@ describe('getCreateArgs', () => {
       new Map<string, string>([
         ['install', 'false'],
         ['use', 'true'],
+        ['cache-binary', 'true'],
         ['cleanup', 'true'],
       ]),
       [
@@ -49,6 +50,7 @@ describe('getCreateArgs', () => {
         ['driver', 'docker'],
         ['install', 'false'],
         ['use', 'true'],
+        ['cache-binary', 'true'],
         ['cleanup', 'true'],
       ]),
       [
@@ -66,6 +68,7 @@ describe('getCreateArgs', () => {
         ['install', 'false'],
         ['use', 'false'],
         ['driver-opts', 'image=moby/buildkit:master\nnetwork=host'],
+        ['cache-binary', 'true'],
         ['cleanup', 'true'],
       ]),
       [
@@ -85,6 +88,7 @@ describe('getCreateArgs', () => {
         ['endpoint', 'tls://foo:1234'],
         ['install', 'false'],
         ['use', 'true'],
+        ['cache-binary', 'true'],
         ['cleanup', 'true'],
       ]),
       [
@@ -104,6 +108,7 @@ describe('getCreateArgs', () => {
         ['endpoint', 'tls://foo:1234'],
         ['install', 'false'],
         ['use', 'true'],
+        ['cache-binary', 'true'],
         ['cleanup', 'true'],
       ]),
       [
@@ -122,6 +127,7 @@ describe('getCreateArgs', () => {
         ['install', 'false'],
         ['use', 'false'],
         ['driver-opts', `"env.no_proxy=localhost,127.0.0.1,.mydomain"`],
+        ['cache-binary', 'true'],
         ['cleanup', 'true'],
       ]),
       [
@@ -139,6 +145,7 @@ describe('getCreateArgs', () => {
         ['install', 'false'],
         ['use', 'false'],
         ['platforms', 'linux/amd64\n"linux/arm64,linux/arm/v7"'],
+        ['cache-binary', 'true'],
         ['cleanup', 'true'],
       ]),
       [
@@ -156,6 +163,7 @@ describe('getCreateArgs', () => {
         ['install', 'false'],
         ['use', 'false'],
         ['driver', 'unknown'],
+        ['cache-binary', 'true'],
         ['cleanup', 'true'],
       ]),
       [
@@ -199,6 +207,7 @@ describe('getAppendArgs', () => {
       new Map<string, string>([
         ['install', 'false'],
         ['use', 'true'],
+        ['cache-binary', 'true'],
         ['cleanup', 'true'],
       ]),
       {
diff --git a/action.yml b/action.yml
index f6ad885..bf6cec9 100644
--- a/action.yml
+++ b/action.yml
@@ -44,6 +44,10 @@ inputs:
   append:
     description: 'Append additional nodes to the builder'
     required: false
+  cache-binary:
+    description: 'Cache buildx binary to GitHub Actions cache backend'
+    default: 'true'
+    required: false
   cleanup:
     description: 'Cleanup temp files and remove builder at the end of a job'
     default: 'true'
diff --git a/src/context.ts b/src/context.ts
index 7360c4b..38dbe3a 100644
--- a/src/context.ts
+++ b/src/context.ts
@@ -20,6 +20,7 @@ export interface Inputs {
   config: string;
   configInline: string;
   append: string;
+  cacheBinary: boolean;
   cleanup: boolean;
 }
 
@@ -37,6 +38,7 @@ export async function getInputs(): Promise<Inputs> {
     config: core.getInput('config'),
     configInline: core.getInput('config-inline'),
     append: core.getInput('append'),
+    cacheBinary: core.getBooleanInput('cache-binary'),
     cleanup: core.getBooleanInput('cleanup')
   };
 }
diff --git a/src/main.ts b/src/main.ts
index 6b66c2e..6e61933 100644
--- a/src/main.ts
+++ b/src/main.ts
@@ -38,11 +38,11 @@ actionsToolkit.run(
         throw new Error(`Cannot build from source without the Docker CLI`);
       }
       await core.group(`Build buildx from source`, async () => {
-        toolPath = await toolkit.buildxInstall.build(inputs.version);
+        toolPath = await toolkit.buildxInstall.build(inputs.version, !inputs.cacheBinary);
       });
     } else if (!(await toolkit.buildx.isAvailable()) || inputs.version) {
       await core.group(`Download buildx from GitHub Releases`, async () => {
-        toolPath = await toolkit.buildxInstall.download(inputs.version || 'latest');
+        toolPath = await toolkit.buildxInstall.download(inputs.version || 'latest', !inputs.cacheBinary);
       });
     }
     if (toolPath) {