diff --git a/.yarnrc.yml b/.yarnrc.yml new file mode 100644 index 0000000..3186f3f --- /dev/null +++ b/.yarnrc.yml @@ -0,0 +1 @@ +nodeLinker: node-modules diff --git a/packages/app/src/components/Root/Root.tsx b/packages/app/src/components/Root/Root.tsx index 28d7342..21dd670 100644 --- a/packages/app/src/components/Root/Root.tsx +++ b/packages/app/src/components/Root/Root.tsx @@ -5,7 +5,7 @@ import ExtensionIcon from '@material-ui/icons/Extension'; import MapIcon from '@material-ui/icons/MyLocation'; import LibraryBooks from '@material-ui/icons/LibraryBooks'; import CreateComponentIcon from '@material-ui/icons/AddCircleOutline'; -import {LogoFull, LogoIcon} from '@internal/plugin-cnoe-ui'; +//import {LogoFull, LogoIcon} from '@internal/plugin-cnoe-ui'; import { Settings as SidebarSettings, UserSettingsSignInAvatar, @@ -20,7 +20,7 @@ import { SidebarPage, SidebarScrollWrapper, SidebarSpace, - useSidebarOpenState, + //useSidebarOpenState, Link, } from '@backstage/core-components'; import MenuIcon from '@material-ui/icons/Menu'; @@ -43,12 +43,12 @@ const useSidebarLogoStyles = makeStyles({ const SidebarLogo = () => { const classes = useSidebarLogoStyles(); - const { isOpen } = useSidebarOpenState(); +// const { isOpen } = useSidebarOpenState(); return (
- {isOpen ? : } + {/*isOpen ? : */}
); diff --git a/packages/backend/package.json b/packages/backend/package.json index 85a58b7..86b2915 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -47,7 +47,6 @@ "@backstage/plugin-search-backend-node": "^1.2.26", "@backstage/plugin-techdocs-backend": "^1.10.8", "@backstage/types": "^1.1.1", - "@internal/backstage-plugin-terraform-backend": "^0.1.0", "@kubernetes/client-node": "~0.20.0", "@roadiehq/backstage-plugin-argo-cd-backend": "3.0.2", "@roadiehq/scaffolder-backend-module-http-request": "^4.3.5", diff --git a/packages/backend/src/index.ts b/packages/backend/src/index.ts index 15b9129..83cf46b 100644 --- a/packages/backend/src/index.ts +++ b/packages/backend/src/index.ts @@ -35,6 +35,6 @@ backend.add( // cnoe plugins //backend.add(authModuleKeycloakOIDCProvider); backend.add(cnoeScaffolderActions); -backend.add(import('@internal/backstage-plugin-terraform-backend')); +//backend.add(import('@internal/backstage-plugin-terraform-backend')); backend.start(); diff --git a/plugins/apache-spark/.eslintrc.js b/plugins/apache-spark/.eslintrc.js deleted file mode 100644 index e2a53a6..0000000 --- a/plugins/apache-spark/.eslintrc.js +++ /dev/null @@ -1 +0,0 @@ -module.exports = require('@backstage/cli/config/eslint-factory')(__dirname); diff --git a/plugins/apache-spark/README.md b/plugins/apache-spark/README.md deleted file mode 100644 index 976aba2..0000000 --- a/plugins/apache-spark/README.md +++ /dev/null @@ -1,13 +0,0 @@ -# apache-spark - -Welcome to the apache-spark plugin! - -_This plugin was created through the Backstage CLI_ - -## Getting started - -Your plugin has been added to the example app in this repository, meaning you'll be able to access it by running `yarn start` in the root directory, and then navigating to [/apache-spark](http://localhost:3000/apache-spark). - -You can also serve the plugin in isolation by running `yarn start` in the plugin directory. -This method of serving the plugin provides quicker iteration speed and a faster startup and hot reloads. -It is only meant for local development, and the setup for it can be found inside the [/dev](./dev) directory. diff --git a/plugins/apache-spark/dev/index.tsx b/plugins/apache-spark/dev/index.tsx deleted file mode 100644 index 5f2b474..0000000 --- a/plugins/apache-spark/dev/index.tsx +++ /dev/null @@ -1,12 +0,0 @@ -import React from 'react'; -import { createDevApp } from '@backstage/dev-utils'; -import { apacheSparkPlugin, ApacheSparkPage } from '../src/plugin'; - -createDevApp() - .registerPlugin(apacheSparkPlugin) - .addPage({ - element: , - title: 'Root Page', - path: '/apache-spark' - }) - .render(); diff --git a/plugins/apache-spark/package.json b/plugins/apache-spark/package.json deleted file mode 100644 index 71639db..0000000 --- a/plugins/apache-spark/package.json +++ /dev/null @@ -1,51 +0,0 @@ -{ - "name": "@internal/plugin-apache-spark", - "version": "0.1.0", - "main": "src/index.ts", - "types": "src/index.ts", - "license": "Apache-2.0", - "private": true, - "publishConfig": { - "access": "public", - "main": "dist/index.esm.js", - "types": "dist/index.d.ts" - }, - "backstage": { - "role": "frontend-plugin" - }, - "sideEffects": false, - "scripts": { - "start": "backstage-cli package start", - "build": "backstage-cli package build", - "lint": "backstage-cli package lint", - "test": "backstage-cli package test", - "clean": "backstage-cli package clean", - "prepack": "backstage-cli package prepack", - "postpack": "backstage-cli package postpack" - }, - "dependencies": { - "@backstage/core-components": "^0.14.8", - "@backstage/core-plugin-api": "^1.9.3", - "@backstage/theme": "^0.5.6", - "@material-ui/core": "^4.9.13", - "@material-ui/icons": "^4.9.1", - "@material-ui/lab": "^4.0.0-alpha.61", - "react-use": "^17.2.4" - }, - "peerDependencies": { - "react": "^16.13.1 || ^17.0.0" - }, - "devDependencies": { - "@backstage/cli": "^0.26.10", - "@backstage/core-app-api": "^1.13.0", - "@backstage/dev-utils": "^1.0.34", - "@backstage/test-utils": "^1.5.7", - "@testing-library/jest-dom": "^5.10.1", - "@testing-library/react": "^12.1.3", - "@testing-library/user-event": "^14.0.0", - "msw": "^1.0.0" - }, - "files": [ - "dist" - ] -} diff --git a/plugins/apache-spark/src/api/index.test.ts b/plugins/apache-spark/src/api/index.test.ts deleted file mode 100644 index 20f775b..0000000 --- a/plugins/apache-spark/src/api/index.test.ts +++ /dev/null @@ -1,113 +0,0 @@ -// import { ApacheSparkClient } from './index'; -// import { ApacheSpark } from './model'; -// -// const mockKubernetesApi = { -// proxy: jest.fn(), -// getClusters: jest.fn(), -// getObjectsByEntity: jest.fn(), -// getWorkloadsByEntity: jest.fn(), -// getCustomObjectsByEntity: jest.fn(), -// }; -// -// describe('ApacheSparkClient', () => { -// let apacheSparkClient: ApacheSparkClient; -// -// beforeEach(() => { -// apacheSparkClient = new ApacheSparkClient(mockKubernetesApi); -// }); -// -// afterEach(() => { -// jest.clearAllMocks(); -// }); -// -// it('should fetch Spark application logs', async () => { -// mockKubernetesApi.proxy.mockResolvedValue({ -// ok: true, -// text: () => { -// return 'logs'; -// }, -// }); -// const logs = await apacheSparkClient.getLogs( -// 'cluster1', -// 'spark-namespace', -// 'spark-pod-name', -// 'abc', -// ); -// expect(logs).toEqual('logs'); -// expect(mockKubernetesApi.proxy).toHaveBeenCalledWith({ -// clusterName: 'cluster1', -// path: '/api/v1/namespaces/spark-namespace/pods/spark-pod-name/log?tailLines=1000&container=abc', -// }); -// }); -// -// it('should throw error if Spark application logs are not fetched', async () => { -// mockKubernetesApi.proxy.mockResolvedValueOnce({ -// status: 500, -// statusText: 'Internal Server Error', -// ok: false, -// text: () => { -// return 'oh noes'; -// }, -// }); -// -// await expect( -// apacheSparkClient.getLogs( -// 'spark-app-name', -// 'spark-namespace', -// 'spark-pod-name', -// 'abc', -// ), -// ).rejects.toEqual( -// 'failed to fetch logs: 500, Internal Server Error, oh noes', -// ); -// }); -// -// // test getSparkApp method -// it('should fetch Spark application', async () => { -// // @ts-ignore -// const mockResponse: ApacheSpark = { -// apiVersion: 'sparkoperator.k8s.io/v1beta2', -// kind: 'SparkApplication', -// metadata: { -// name: 'spark-app-name', -// namespace: 'spark-namespace', -// labels: { -// app: 'spark-app-name', -// }, -// creationTimestamp: '2021-01-01T00:00:00Z', -// }, -// spec: { -// image: 'abc', -// mainApplicationFile: 'main.py', -// mode: 'cluster', -// sparkVersion: 'v3.1.1.', -// type: 'Python', -// driver: { -// cores: 1, -// }, -// executor: { -// cores: 1, -// }, -// }, -// status: { -// applicationState: { -// state: 'RUNNING', -// }, -// }, -// }; -// -// mockKubernetesApi.proxy.mockResolvedValue({ -// ok: true, -// text: () => { -// return JSON.stringify(mockResponse); -// }, -// }); -// -// const application = await apacheSparkClient.getSparkApp( -// 'spark-app-name', -// 'spark-namespace', -// 'abc', -// ); -// expect(application).toEqual(mockResponse); -// }); -// }); diff --git a/plugins/apache-spark/src/api/index.ts b/plugins/apache-spark/src/api/index.ts deleted file mode 100644 index cda9454..0000000 --- a/plugins/apache-spark/src/api/index.ts +++ /dev/null @@ -1,176 +0,0 @@ -import { createApiRef } from '@backstage/core-plugin-api'; -import { ApacheSpark, ApacheSparkList, Pod } from './model'; -import { KubernetesApi } from '@backstage/plugin-kubernetes'; - -export const apacheSparkApiRef = createApiRef({ - id: 'plugin.apachespark', -}); - -const API_VERSION = 'sparkoperator.k8s.io/v1beta2'; -const SPARK_APP_PLURAL = 'sparkapplications'; -const K8s_API_TIMEOUT = 'timeoutSeconds'; - -export interface ApacheSparkApi { - getSparkApps( - clusterName: string | undefined, - namespace: string | undefined, - labels: string | undefined, - ): Promise; - - getSparkApp( - clusterName: string | undefined, - namespace: string | undefined, - name: string, - ): Promise; - - getLogs( - clusterName: string | undefined, - namespace: string | undefined, - podName: string, - containerName?: string | undefined, - tailLine?: number, - ): Promise; - - getContainers( - clusterName: string | undefined, - namespace: string | undefined, - podName: string, - ): Promise; -} - -export class ApacheSparkClient implements ApacheSparkApi { - private kubernetesApi: KubernetesApi; - constructor(kubernetesApi: KubernetesApi) { - this.kubernetesApi = kubernetesApi; - } - async getSparkApps( - clusterName: string | undefined, - namespace: string | undefined, - labels: string | undefined, - ): Promise { - const ns = namespace !== undefined ? namespace : 'default'; - const path = `/apis/${API_VERSION}/namespaces/${ns}/${SPARK_APP_PLURAL}`; - const query = new URLSearchParams({ - [K8s_API_TIMEOUT]: '30', - }); - if (labels) { - query.set('labelSelector', labels); - } - const resp = await this.kubernetesApi.proxy({ - clusterName: - clusterName !== undefined ? clusterName : await this.getFirstCluster(), - path: `${path}?${query.toString()}`, - }); - - if (!resp.ok) { - return Promise.reject( - `failed to fetch resources: ${resp.status}, ${ - resp.statusText - }, ${await resp.text()}`, - ); - } - const out = JSON.parse(await resp.text()); - this.removeManagedField(out); - return out; - } - - async getSparkApp( - clusterName: string | undefined, - namespace: string | undefined, - name: string, - ): Promise { - const ns = namespace !== undefined ? namespace : 'default'; - const path = `/apis/${API_VERSION}/namespaces/${ns}/${SPARK_APP_PLURAL}/${name}`; - const resp = await this.kubernetesApi.proxy({ - clusterName: - clusterName !== undefined ? clusterName : await this.getFirstCluster(), - path: `${path}`, - }); - if (!resp.ok) { - return Promise.reject( - `failed to fetch resources: ${resp.status}, ${ - resp.statusText - }, ${await resp.text()}`, - ); - } - const out = JSON.parse(await resp.text()); - this.removeManagedField(out); - return out; - } - - async getLogs( - clusterName: string | undefined, - namespace: string | undefined, - podName: string, - containerName: string | undefined, - tailLine: number = 1000, - ): Promise { - const ns = namespace !== undefined ? namespace : 'default'; - const path = `/api/v1/namespaces/${ns}/pods/${podName}/log`; - const query = new URLSearchParams({ - tailLines: tailLine.toString(), - }); - if (containerName) { - query.set('container', containerName); - } - - const resp = await this.kubernetesApi.proxy({ - clusterName: - clusterName !== undefined ? clusterName : await this.getFirstCluster(), - path: `${path}?${query.toString()}`, - }); - if (!resp.ok) { - return Promise.reject( - `failed to fetch logs: ${resp.status}, ${ - resp.statusText - }, ${await resp.text()}`, - ); - } - return resp.text(); - } - - async getContainers( - clusterName: string | undefined, - namespace: string | undefined, - podName: string, - ): Promise { - const ns = namespace !== undefined ? namespace : 'default'; - const path = `/api/v1/namespaces/${ns}/pods/${podName}`; - const query = new URLSearchParams({ - [K8s_API_TIMEOUT]: '30', - }); - const resp = await this.kubernetesApi.proxy({ - clusterName: - clusterName !== undefined ? clusterName : await this.getFirstCluster(), - path: `${path}?${query.toString()}`, - }); - if (!resp.ok) { - throw new Error( - `failed to fetch logs: ${resp.status}, ${ - resp.statusText - }, ${await resp.text()}`, - ); - } - const pod = JSON.parse(await resp.text()) as Pod; - return pod.spec.containers.map(c => c.name); - } - - async getFirstCluster(): Promise { - const clusters = await this.kubernetesApi.getClusters(); - if (clusters.length > 0) { - return Promise.resolve(clusters[0].name); - } - return Promise.reject('no clusters found in configuration'); - } - - removeManagedField(spark: any) { - if (spark.metadata?.hasOwnProperty('managedFields')) { - delete spark.metadata.managedFields; - } - if (spark.items) { - for (const i of spark.items) { - this.removeManagedField(i); - } - } - } -} diff --git a/plugins/apache-spark/src/api/model.ts b/plugins/apache-spark/src/api/model.ts deleted file mode 100644 index 1d6455c..0000000 --- a/plugins/apache-spark/src/api/model.ts +++ /dev/null @@ -1,100 +0,0 @@ -export type Metadata = { - name: string; - namespace?: string; - labels?: Record; - annotations?: Record; - creationTimestamp: string; - managedFields?: any; -}; - -export type Spec = { - arguments?: string[]; - batchScheduler?: string; - driver: { - coreLimit?: string; - coreRequest?: string; - cores?: number; - gpu?: { - name: string; - quantity: number; - }; - labels?: Record; - memory?: string; - memoryOverhead?: string; - podName?: string; - schedulerName?: string; - serviceAccount?: string; - }; - executor: { - coreLimit?: string; - coreRequest?: string; - cores?: number; - gpu?: { - name: string; - quantity: number; - }; - instances?: number; - labels?: Record; - memory?: string; - memoryOverhead?: string; - schedulerName?: string; - serviceAccount?: string; - }; - image: string; - mainClass?: string; - mainApplicationFile?: string; - mode: string; - pythonVersion?: string; - sparkVersion: string; - type: string; -}; - -export type Status = { - applicationState: { - errorMessage?: string; - state: string; - }; - driverInfo?: { - podName: string; - webUIAddress: string; - webUIIngressAddress: string; - webUIIngressName: string; - webUIPort: string; - webUIServiceName: string; - }; - executionAttempts?: number; - executorState?: { [key: string]: string }; - lastSubmissionAttemptTime?: string; - sparkApplicationId?: string; - submissionAttempts?: number; - submissionID?: string; - terminationTime?: string; -}; - -export type ApacheSpark = { - apiVersion: string; - kind: string; - metadata: Metadata; - spec: Spec; - status: Status; -}; - -export type ApacheSparkList = { - apiVersion: string; - kind: string; - items?: ApacheSpark[]; -}; - -export type Pod = { - apiVersion: string; - kind: string; - metadata: Metadata; - spec: PodSpec; -}; - -export type PodSpec = { - containers: { - image: string; - name: string; - }[]; -}; diff --git a/plugins/apache-spark/src/components/ApacheSparkLogs/ApacheSparkLogs.test.tsx b/plugins/apache-spark/src/components/ApacheSparkLogs/ApacheSparkLogs.test.tsx deleted file mode 100644 index 01d3ade..0000000 --- a/plugins/apache-spark/src/components/ApacheSparkLogs/ApacheSparkLogs.test.tsx +++ /dev/null @@ -1,83 +0,0 @@ -import React from 'react'; -import { render, screen } from '@testing-library/react'; -import { useApi } from '@backstage/core-plugin-api'; -import { useEntity } from '@backstage/plugin-catalog-react'; -import useAsync from 'react-use/lib/useAsync'; -import { ApacheSpark } from '../../api/model'; -import { ApacheSparkDriverLogs } from './ApacheSparkLogs'; -import { - APACHE_SPARK_LABEL_SELECTOR_ANNOTATION, - CLUSTER_NAME_ANNOTATION, - K8S_NAMESPACE_ANNOTATION, -} from '../../consts'; - -jest.mock('@backstage/core-plugin-api'); -jest.mock('react-use/lib/useAsync'); -jest.mock('@backstage/plugin-catalog-react'); - -jest.mock('@backstage/core-components', () => ({ - LogViewer: (props: { text: string }) => { - return
{props.text}
; - }, -})); - -describe('ApacheSparkDriverLogs', () => { - const mockUseApi = useApi as jest.MockedFunction; - const mockUseAsync = useAsync as jest.MockedFunction; - const mockUseEntity = useEntity as jest.MockedFunction; - const mockGetLogs = jest.fn(); - const mockSparkApp = { - status: { - driverInfo: { - podName: 'test-pod', - }, - }, - } as ApacheSpark; - - beforeEach(() => { - mockUseApi.mockReturnValue({ - getLogs: mockGetLogs, - }); - mockUseEntity.mockReturnValue({ - entity: { - apiVersion: 'version', - kind: 'kind', - metadata: { - name: 'name', - namespace: 'ns1', - annotations: { - [K8S_NAMESPACE_ANNOTATION]: 'k8s-ns', - [CLUSTER_NAME_ANNOTATION]: 'my-cluster', - [APACHE_SPARK_LABEL_SELECTOR_ANNOTATION]: 'env=test', - }, - }, - }, - }); - }); - - afterEach(() => { - jest.clearAllMocks(); - }); - - it('should render error message if there is an error', () => { - mockUseAsync.mockReturnValue({ - value: undefined, - loading: false, - error: new Error('Test error'), - }); - - render(); - expect(screen.getByText('Error: Test error')).toBeInTheDocument(); - expect(screen.getByRole('alert')).toBeInTheDocument(); - }); - - it('should render the log viewer with the fetched logs', async () => { - mockUseAsync.mockReturnValue({ - value: 'test logs', - loading: false, - error: undefined, - }); - render(); - expect(screen.getByText('test logs')).toBeInTheDocument(); - }); -}); diff --git a/plugins/apache-spark/src/components/ApacheSparkLogs/ApacheSparkLogs.tsx b/plugins/apache-spark/src/components/ApacheSparkLogs/ApacheSparkLogs.tsx deleted file mode 100644 index e892856..0000000 --- a/plugins/apache-spark/src/components/ApacheSparkLogs/ApacheSparkLogs.tsx +++ /dev/null @@ -1,100 +0,0 @@ -import { useApi } from '@backstage/core-plugin-api'; -import { apacheSparkApiRef } from '../../api'; -import useAsync from 'react-use/lib/useAsync'; -import { ApacheSpark } from '../../api/model'; -import { - LogViewer, - Progress, - Select, - SelectedItems, - SelectItem, -} from '@backstage/core-components'; -import Alert from '@material-ui/lab/Alert'; -import React, { useEffect, useState } from 'react'; -import { useEntity } from '@backstage/plugin-catalog-react'; -import { getAnnotationValues } from '../utils'; - -export const ApacheSparkDriverLogs = (props: { sparkApp: ApacheSpark }) => { - const apiClient = useApi(apacheSparkApiRef); - const { entity } = useEntity(); - const { ns, clusterName } = getAnnotationValues(entity); - - const { value, loading, error } = useAsync(async (): Promise => { - return await apiClient.getLogs( - clusterName, - ns, - props.sparkApp.status.driverInfo?.podName!, - 'spark-kubernetes-driver', - ); - }, [props]); - if (loading) { - return ; - } else if (error) { - return {`${error}`}; - } - return ; -}; - -const ExecutorLogs = (props: { name: string }) => { - const apiClient = useApi(apacheSparkApiRef); - const { entity } = useEntity(); - const [logs, setLogs] = useState(''); - const { ns, clusterName } = getAnnotationValues(entity); - - useEffect(() => { - async function getLogs() { - try { - const val = await apiClient.getLogs( - clusterName, - ns, - props.name, - 'spark-kubernetes-executor', - ); - setLogs(val); - } catch (e) { - if (typeof e === 'string') { - setLogs(e); - } - } - } - if (props.name !== '') { - getLogs(); - } - }, [apiClient, clusterName, ns, props]); - - return ; -}; - -export const ApacheSparkExecutorLogs = (props: { sparkApp: ApacheSpark }) => { - const [selected, setSelected] = useState(''); - if (props.sparkApp.status.applicationState.state !== 'RUNNING') { - return ( - - Executor logs are only available for Spark Applications in RUNNING state - - ); - } - const executors: SelectItem[] = [{ label: '', value: '' }]; - for (const key in props.sparkApp.status.executorState) { - if (props.sparkApp.status.executorState.hasOwnProperty(key)) { - executors.push({ label: key, value: key }); - } - } - - const handleChange = (item: SelectedItems) => { - if (typeof item === 'string' && item !== '') { - setSelected(item); - } - }; - return ( - <> -