From 2862d9cd3c0201ffb05c2350bea4e3676ce37579 Mon Sep 17 00:00:00 2001 From: richardrobertreitz Date: Sat, 15 Mar 2025 06:25:26 +0000 Subject: [PATCH 1/7] Update .github/workflows/build-and-push.yaml --- .github/workflows/build-and-push.yaml | 52 +++++++++++++++++---------- 1 file changed, 33 insertions(+), 19 deletions(-) diff --git a/.github/workflows/build-and-push.yaml b/.github/workflows/build-and-push.yaml index decb3c6..72f9612 100644 --- a/.github/workflows/build-and-push.yaml +++ b/.github/workflows/build-and-push.yaml @@ -1,37 +1,51 @@ -name: Create and publish a Docker image +name: ci on: push jobs: - docker: - runs-on: docker - - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - submodules: recursive + build: + runs-on: ubuntu-22.04 - - name: Repository meta + steps: + - + name: Repository meta id: repository run: | registry=${{ github.server_url }} registry=${registry##http*://} echo "registry=${registry}" >> "$GITHUB_OUTPUT" + echo "registry=${registry}" repository="$(echo "${{ github.repository }}" | tr '[:upper:]' '[:lower:]')" echo "repository=${repository}" >> "$GITHUB_OUTPUT" - - - name: Docker meta + echo "repository=${repository}" + - + name: Docker meta uses: docker/metadata-action@v5 id: docker with: images: ${{ steps.repository.outputs.registry }}/${{ steps.repository.outputs.repository }} - - - name: Build and push - uses: https://codeberg.org/umglurf/kaniko-action@main + - + name: Login to registry + uses: docker/login-action@v3 + with: + registry: ${{ steps.repository.outputs.registry }} + username: ${{ github.repository }} + password: ${{ secrets.PACKAGES_TOKEN }} + - + name: Set up QEMU + uses: docker/setup-qemu-action@v3 + - + name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + with: + buildkitd-flags: '--allow-insecure-entitlement network.host' + driver-opts: network=host + - + name: Build and push + uses: docker/build-push-action@v6 with: - credentials: | - ${{ steps.repository.outputs.registry }}=${{ github.repository }}:${{ secrets.PACKAGES_TOKEN }} - destinations: | - ${{ steps.docker.outputs.tags }} push: true + allow: network.host + network: host + platforms: linux/amd64,linux/arm64 + tags: ${{ steps.docker.outputs.tags }} From 1f6a768b789f895e9db3ca718c24a81ff4ea0414 Mon Sep 17 00:00:00 2001 From: richardrobertreitz Date: Sat, 15 Mar 2025 06:27:35 +0000 Subject: [PATCH 2/7] Update .github/workflows/build-and-push.yaml --- .github/workflows/build-and-push.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-and-push.yaml b/.github/workflows/build-and-push.yaml index 72f9612..86cec88 100644 --- a/.github/workflows/build-and-push.yaml +++ b/.github/workflows/build-and-push.yaml @@ -29,8 +29,8 @@ jobs: uses: docker/login-action@v3 with: registry: ${{ steps.repository.outputs.registry }} - username: ${{ github.repository }} - password: ${{ secrets.PACKAGES_TOKEN }} + username: ${{ secrets.REGISTRY_USERNAME }} + password: ${{ secrets.REGISTRY_PASSWORD }} - name: Set up QEMU uses: docker/setup-qemu-action@v3 From 0b2c70ec3c81cb4c341be12fda33179449c3c99d Mon Sep 17 00:00:00 2001 From: richardrobertreitz Date: Sat, 15 Mar 2025 07:08:26 +0000 Subject: [PATCH 3/7] Update .github/workflows/build-and-push.yaml --- .github/workflows/build-and-push.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-and-push.yaml b/.github/workflows/build-and-push.yaml index 86cec88..6be5e79 100644 --- a/.github/workflows/build-and-push.yaml +++ b/.github/workflows/build-and-push.yaml @@ -47,5 +47,5 @@ jobs: push: true allow: network.host network: host - platforms: linux/amd64,linux/arm64 + platforms: linux/amd64 tags: ${{ steps.docker.outputs.tags }} From 8c27f2f322f4138700847e04f3cc966a9f0bb496 Mon Sep 17 00:00:00 2001 From: richardrobertreitz Date: Sat, 15 Mar 2025 08:21:54 +0000 Subject: [PATCH 4/7] Update .github/workflows/build-and-push.yaml --- .github/workflows/build-and-push.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-and-push.yaml b/.github/workflows/build-and-push.yaml index 6be5e79..9bb12b8 100644 --- a/.github/workflows/build-and-push.yaml +++ b/.github/workflows/build-and-push.yaml @@ -29,8 +29,8 @@ jobs: uses: docker/login-action@v3 with: registry: ${{ steps.repository.outputs.registry }} - username: ${{ secrets.REGISTRY_USERNAME }} - password: ${{ secrets.REGISTRY_PASSWORD }} + username: ${{ secrets.PACKAGES_USER }} + password: ${{ secrets.PACKAGES_TOKEN }} - name: Set up QEMU uses: docker/setup-qemu-action@v3 @@ -47,5 +47,5 @@ jobs: push: true allow: network.host network: host - platforms: linux/amd64 + platforms: linux/amd64,linux/arm64 tags: ${{ steps.docker.outputs.tags }} From b09f1fd853f42ed69f4948d9c2659a9554b11efc Mon Sep 17 00:00:00 2001 From: richardrobertreitz Date: Sat, 15 Mar 2025 12:09:46 +0000 Subject: [PATCH 5/7] Update .github/workflows/build-and-push.yaml --- .github/workflows/build-and-push.yaml | 150 +++++++++++++++++--------- 1 file changed, 102 insertions(+), 48 deletions(-) diff --git a/.github/workflows/build-and-push.yaml b/.github/workflows/build-and-push.yaml index 9bb12b8..bb64a6a 100644 --- a/.github/workflows/build-and-push.yaml +++ b/.github/workflows/build-and-push.yaml @@ -1,51 +1,105 @@ -name: ci +# Stage 1 - Create yarn install skeleton layer +FROM node:20.18.1 AS packages -on: push +WORKDIR /app +COPY package.json yarn.lock ./ -jobs: - build: - runs-on: ubuntu-22.04 +COPY packages packages - steps: - - - name: Repository meta - id: repository - run: | - registry=${{ github.server_url }} - registry=${registry##http*://} - echo "registry=${registry}" >> "$GITHUB_OUTPUT" - echo "registry=${registry}" - repository="$(echo "${{ github.repository }}" | tr '[:upper:]' '[:lower:]')" - echo "repository=${repository}" >> "$GITHUB_OUTPUT" - echo "repository=${repository}" - - - name: Docker meta - uses: docker/metadata-action@v5 - id: docker - with: - images: ${{ steps.repository.outputs.registry }}/${{ steps.repository.outputs.repository }} - - - name: Login to registry - uses: docker/login-action@v3 - with: - registry: ${{ steps.repository.outputs.registry }} - username: ${{ secrets.PACKAGES_USER }} - password: ${{ secrets.PACKAGES_TOKEN }} - - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - with: - buildkitd-flags: '--allow-insecure-entitlement network.host' - driver-opts: network=host - - - name: Build and push - uses: docker/build-push-action@v6 - with: - push: true - allow: network.host - network: host - platforms: linux/amd64,linux/arm64 - tags: ${{ steps.docker.outputs.tags }} +# Comment this out if you don't have any internal plugins +COPY plugins plugins + +RUN find packages \! -name "package.json" -mindepth 2 -maxdepth 2 -exec rm -rf {} \+ + +# Stage 2 - Install dependencies and build packages +FROM node:20.18.1 AS build + +# Required for arm64 +RUN apt update -y +RUN apt install -y python3 make gcc build-essential bash + +USER node +WORKDIR /app + +COPY --from=packages --chown=node:node /app . + +RUN --mount=type=cache,target=/home/node/.cache/yarn,sharing=locked,uid=1000,gid=1000 \ + yarn install --frozen-lockfile --network-timeout 600000 + +COPY --chown=node:node . . + +RUN yarn tsc +RUN yarn --cwd packages/backend build +# If you have not yet migrated to package roles, use the following command instead: +# RUN yarn --cwd packages/backend backstage-cli backend:bundle --build-dependencies + +RUN mkdir packages/backend/dist/skeleton packages/backend/dist/bundle \ + && tar xzf packages/backend/dist/skeleton.tar.gz -C packages/backend/dist/skeleton \ + && tar xzf packages/backend/dist/bundle.tar.gz -C packages/backend/dist/bundle + +# Stage 3 - Build the actual backend image and install production dependencies +FROM node:20.18.1 + +# Install isolate-vm dependencies, these are needed by the @backstage/plugin-scaffolder-backend. +# Install packages needed to get utility binaries +RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ + --mount=type=cache,target=/var/lib/apt,sharing=locked \ + apt-get update && \ + apt-get install -y --no-install-recommends python3 python3-pip python3-venv g++ build-essential ca-certificates curl + +RUN yarn config set python /usr/bin/python3 + +# Add kubectl for the kube apply plugin. +# Add cnoe-cli +# Add mkdocs for the TechDocs plugin. +RUN if test "$(uname -m)" = "x86_64"; \ + then \ + curl -L -o /usr/local/bin/kubectl https://dl.k8s.io/release/v1.29.9/bin/linux/amd64/kubectl; \ + fi +RUN if test "$(uname -m)" != "x86_64"; \ + then \ + curl -L -o /usr/local/bin/kubectl https://dl.k8s.io/release/v1.29.9/bin/linux/arm64/kubectl; \ + fi +RUN chmod +x /usr/local/bin/kubectl + +RUN curl -L -O https://github.com/cnoe-io/cnoe-cli/releases/download/v0.1.0/cnoe_Linux_x86_64.tar.gz && \ + curl -L -O https://github.com/cnoe-io/cnoe-cli/releases/download/v0.1.0/checksums.txt && \ + sha256sum -c --strict --status --ignore-missing checksums.txt && \ + tar -xzf cnoe_Linux_x86_64.tar.gz && \ + mv cnoe /usr/bin/cnoe-cli && \ + chmod +x /usr/bin/cnoe-cli && \ + rm checksums.txt cnoe_Linux_x86_64.tar.gz +COPY ./cnoe-wrapper.sh /usr/bin/cnoe +RUN chmod +x /usr/bin/cnoe + +ENV VIRTUAL_ENV=/opt/venv +RUN python3 -m venv $VIRTUAL_ENV +ENV PATH="$VIRTUAL_ENV/bin:$PATH" +RUN pip3 install 'mkdocs-techdocs-core==1.4.2' + +# From here on we use the least-privileged `node` user to run the backend. +USER node + +# This should create the app dir as `node`. +# If it is instead created as `root` then the `tar` command below will +# fail: `can't create directory 'packages/': Permission denied`. +# If this occurs, then ensure BuildKit is enabled (`DOCKER_BUILDKIT=1`) +# so the app dir is correctly created as `node`. +WORKDIR /app + +# Copy the install dependencies from the build stage and context +COPY --from=build --chown=node:node /app/yarn.lock /app/package.json /app/packages/backend/dist/skeleton/ ./ + +RUN --mount=type=cache,target=/home/node/.cache/yarn,sharing=locked,uid=1000,gid=1000 \ + yarn install --frozen-lockfile --production --network-timeout 600000 + +# Copy the built packages from the build stage +COPY --from=build --chown=node:node /app/packages/backend/dist/bundle/ ./ + +# Copy any other files that we need at runtime +COPY --chown=node:node app-config.yaml ./ + +# This switches many Node.js dependencies to production mode. +ENV NODE_ENV production + +CMD ["node", "packages/backend", "--config", "app-config.yaml"] From b66b6f953ae803f161c2ee2febbc885766534621 Mon Sep 17 00:00:00 2001 From: richardrobertreitz Date: Sat, 15 Mar 2025 12:11:11 +0000 Subject: [PATCH 6/7] Update .github/workflows/build-and-push.yaml --- .github/workflows/build-and-push.yaml | 150 +++++++++----------------- 1 file changed, 48 insertions(+), 102 deletions(-) diff --git a/.github/workflows/build-and-push.yaml b/.github/workflows/build-and-push.yaml index bb64a6a..9bb12b8 100644 --- a/.github/workflows/build-and-push.yaml +++ b/.github/workflows/build-and-push.yaml @@ -1,105 +1,51 @@ -# Stage 1 - Create yarn install skeleton layer -FROM node:20.18.1 AS packages +name: ci -WORKDIR /app -COPY package.json yarn.lock ./ +on: push -COPY packages packages +jobs: + build: + runs-on: ubuntu-22.04 -# Comment this out if you don't have any internal plugins -COPY plugins plugins - -RUN find packages \! -name "package.json" -mindepth 2 -maxdepth 2 -exec rm -rf {} \+ - -# Stage 2 - Install dependencies and build packages -FROM node:20.18.1 AS build - -# Required for arm64 -RUN apt update -y -RUN apt install -y python3 make gcc build-essential bash - -USER node -WORKDIR /app - -COPY --from=packages --chown=node:node /app . - -RUN --mount=type=cache,target=/home/node/.cache/yarn,sharing=locked,uid=1000,gid=1000 \ - yarn install --frozen-lockfile --network-timeout 600000 - -COPY --chown=node:node . . - -RUN yarn tsc -RUN yarn --cwd packages/backend build -# If you have not yet migrated to package roles, use the following command instead: -# RUN yarn --cwd packages/backend backstage-cli backend:bundle --build-dependencies - -RUN mkdir packages/backend/dist/skeleton packages/backend/dist/bundle \ - && tar xzf packages/backend/dist/skeleton.tar.gz -C packages/backend/dist/skeleton \ - && tar xzf packages/backend/dist/bundle.tar.gz -C packages/backend/dist/bundle - -# Stage 3 - Build the actual backend image and install production dependencies -FROM node:20.18.1 - -# Install isolate-vm dependencies, these are needed by the @backstage/plugin-scaffolder-backend. -# Install packages needed to get utility binaries -RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ - --mount=type=cache,target=/var/lib/apt,sharing=locked \ - apt-get update && \ - apt-get install -y --no-install-recommends python3 python3-pip python3-venv g++ build-essential ca-certificates curl - -RUN yarn config set python /usr/bin/python3 - -# Add kubectl for the kube apply plugin. -# Add cnoe-cli -# Add mkdocs for the TechDocs plugin. -RUN if test "$(uname -m)" = "x86_64"; \ - then \ - curl -L -o /usr/local/bin/kubectl https://dl.k8s.io/release/v1.29.9/bin/linux/amd64/kubectl; \ - fi -RUN if test "$(uname -m)" != "x86_64"; \ - then \ - curl -L -o /usr/local/bin/kubectl https://dl.k8s.io/release/v1.29.9/bin/linux/arm64/kubectl; \ - fi -RUN chmod +x /usr/local/bin/kubectl - -RUN curl -L -O https://github.com/cnoe-io/cnoe-cli/releases/download/v0.1.0/cnoe_Linux_x86_64.tar.gz && \ - curl -L -O https://github.com/cnoe-io/cnoe-cli/releases/download/v0.1.0/checksums.txt && \ - sha256sum -c --strict --status --ignore-missing checksums.txt && \ - tar -xzf cnoe_Linux_x86_64.tar.gz && \ - mv cnoe /usr/bin/cnoe-cli && \ - chmod +x /usr/bin/cnoe-cli && \ - rm checksums.txt cnoe_Linux_x86_64.tar.gz -COPY ./cnoe-wrapper.sh /usr/bin/cnoe -RUN chmod +x /usr/bin/cnoe - -ENV VIRTUAL_ENV=/opt/venv -RUN python3 -m venv $VIRTUAL_ENV -ENV PATH="$VIRTUAL_ENV/bin:$PATH" -RUN pip3 install 'mkdocs-techdocs-core==1.4.2' - -# From here on we use the least-privileged `node` user to run the backend. -USER node - -# This should create the app dir as `node`. -# If it is instead created as `root` then the `tar` command below will -# fail: `can't create directory 'packages/': Permission denied`. -# If this occurs, then ensure BuildKit is enabled (`DOCKER_BUILDKIT=1`) -# so the app dir is correctly created as `node`. -WORKDIR /app - -# Copy the install dependencies from the build stage and context -COPY --from=build --chown=node:node /app/yarn.lock /app/package.json /app/packages/backend/dist/skeleton/ ./ - -RUN --mount=type=cache,target=/home/node/.cache/yarn,sharing=locked,uid=1000,gid=1000 \ - yarn install --frozen-lockfile --production --network-timeout 600000 - -# Copy the built packages from the build stage -COPY --from=build --chown=node:node /app/packages/backend/dist/bundle/ ./ - -# Copy any other files that we need at runtime -COPY --chown=node:node app-config.yaml ./ - -# This switches many Node.js dependencies to production mode. -ENV NODE_ENV production - -CMD ["node", "packages/backend", "--config", "app-config.yaml"] + steps: + - + name: Repository meta + id: repository + run: | + registry=${{ github.server_url }} + registry=${registry##http*://} + echo "registry=${registry}" >> "$GITHUB_OUTPUT" + echo "registry=${registry}" + repository="$(echo "${{ github.repository }}" | tr '[:upper:]' '[:lower:]')" + echo "repository=${repository}" >> "$GITHUB_OUTPUT" + echo "repository=${repository}" + - + name: Docker meta + uses: docker/metadata-action@v5 + id: docker + with: + images: ${{ steps.repository.outputs.registry }}/${{ steps.repository.outputs.repository }} + - + name: Login to registry + uses: docker/login-action@v3 + with: + registry: ${{ steps.repository.outputs.registry }} + username: ${{ secrets.PACKAGES_USER }} + password: ${{ secrets.PACKAGES_TOKEN }} + - + name: Set up QEMU + uses: docker/setup-qemu-action@v3 + - + name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + with: + buildkitd-flags: '--allow-insecure-entitlement network.host' + driver-opts: network=host + - + name: Build and push + uses: docker/build-push-action@v6 + with: + push: true + allow: network.host + network: host + platforms: linux/amd64,linux/arm64 + tags: ${{ steps.docker.outputs.tags }} From 3e4f90e1f5c0cbbda511a05c496d7ed445c0c3f9 Mon Sep 17 00:00:00 2001 From: richardrobertreitz Date: Sat, 15 Mar 2025 12:11:54 +0000 Subject: [PATCH 7/7] Update Dockerfile --- Dockerfile | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index 90c8337..bb64a6a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,9 +14,9 @@ RUN find packages \! -name "package.json" -mindepth 2 -maxdepth 2 -exec rm -rf { # Stage 2 - Install dependencies and build packages FROM node:20.18.1 AS build -# Required for macOS +# Required for arm64 RUN apt update -y -RUN apt install -y python3 make gcc build-essential +RUN apt install -y python3 make gcc build-essential bash USER node WORKDIR /app @@ -24,7 +24,7 @@ WORKDIR /app COPY --from=packages --chown=node:node /app . RUN --mount=type=cache,target=/home/node/.cache/yarn,sharing=locked,uid=1000,gid=1000 \ - yarn install --network-timeout 600000 + yarn install --frozen-lockfile --network-timeout 600000 COPY --chown=node:node . . @@ -52,7 +52,15 @@ RUN yarn config set python /usr/bin/python3 # Add kubectl for the kube apply plugin. # Add cnoe-cli # Add mkdocs for the TechDocs plugin. -RUN curl -L -o /usr/local/bin/kubectl https://dl.k8s.io/release/v1.29.9/bin/linux/amd64/kubectl && chmod +x /usr/local/bin/kubectl +RUN if test "$(uname -m)" = "x86_64"; \ + then \ + curl -L -o /usr/local/bin/kubectl https://dl.k8s.io/release/v1.29.9/bin/linux/amd64/kubectl; \ + fi +RUN if test "$(uname -m)" != "x86_64"; \ + then \ + curl -L -o /usr/local/bin/kubectl https://dl.k8s.io/release/v1.29.9/bin/linux/arm64/kubectl; \ + fi +RUN chmod +x /usr/local/bin/kubectl RUN curl -L -O https://github.com/cnoe-io/cnoe-cli/releases/download/v0.1.0/cnoe_Linux_x86_64.tar.gz && \ curl -L -O https://github.com/cnoe-io/cnoe-cli/releases/download/v0.1.0/checksums.txt && \ @@ -83,7 +91,7 @@ WORKDIR /app COPY --from=build --chown=node:node /app/yarn.lock /app/package.json /app/packages/backend/dist/skeleton/ ./ RUN --mount=type=cache,target=/home/node/.cache/yarn,sharing=locked,uid=1000,gid=1000 \ - yarn install --production --network-timeout 600000 + yarn install --frozen-lockfile --production --network-timeout 600000 # Copy the built packages from the build stage COPY --from=build --chown=node:node /app/packages/backend/dist/bundle/ ./