diff --git a/plugins/apache-spark/.eslintrc.js b/plugins/apache-spark/.eslintrc.js new file mode 100644 index 0000000..998aac2 --- /dev/null +++ b/plugins/apache-spark/.eslintrc.js @@ -0,0 +1,3 @@ +module.exports = require('@backstage/cli/config/eslint-factory')(__dirname, { + extends: ['prettier'], +}); diff --git a/plugins/apache-spark/.prettierignore b/plugins/apache-spark/.prettierignore new file mode 100644 index 0000000..5498e0f --- /dev/null +++ b/plugins/apache-spark/.prettierignore @@ -0,0 +1,2 @@ +build +coverage diff --git a/plugins/apache-spark/README.md b/plugins/apache-spark/README.md new file mode 100644 index 0000000..976aba2 --- /dev/null +++ b/plugins/apache-spark/README.md @@ -0,0 +1,13 @@ +# apache-spark + +Welcome to the apache-spark plugin! + +_This plugin was created through the Backstage CLI_ + +## Getting started + +Your plugin has been added to the example app in this repository, meaning you'll be able to access it by running `yarn start` in the root directory, and then navigating to [/apache-spark](http://localhost:3000/apache-spark). + +You can also serve the plugin in isolation by running `yarn start` in the plugin directory. +This method of serving the plugin provides quicker iteration speed and a faster startup and hot reloads. +It is only meant for local development, and the setup for it can be found inside the [/dev](./dev) directory. diff --git a/plugins/apache-spark/dev/index.tsx b/plugins/apache-spark/dev/index.tsx new file mode 100644 index 0000000..5f2b474 --- /dev/null +++ b/plugins/apache-spark/dev/index.tsx @@ -0,0 +1,12 @@ +import React from 'react'; +import { createDevApp } from '@backstage/dev-utils'; +import { apacheSparkPlugin, ApacheSparkPage } from '../src/plugin'; + +createDevApp() + .registerPlugin(apacheSparkPlugin) + .addPage({ + element: , + title: 'Root Page', + path: '/apache-spark' + }) + .render(); diff --git a/plugins/apache-spark/package.json b/plugins/apache-spark/package.json new file mode 100644 index 0000000..28ec3df --- /dev/null +++ b/plugins/apache-spark/package.json @@ -0,0 +1,55 @@ +{ + "name": "@internal/plugin-apache-spark", + "version": "0.1.0", + "main": "src/index.ts", + "types": "src/index.ts", + "license": "Apache-2.0", + "private": true, + "publishConfig": { + "access": "public", + "main": "dist/index.esm.js", + "types": "dist/index.d.ts" + }, + "backstage": { + "role": "frontend-plugin" + }, + "scripts": { + "start": "backstage-cli package start", + "build": "backstage-cli package build", + "lint": "backstage-cli package lint", + "test": "backstage-cli package test", + "clean": "backstage-cli package clean", + "prepack": "backstage-cli package prepack", + "postpack": "backstage-cli package postpack" + }, + "dependencies": { + "@backstage/core-components": "^0.13.1", + "@backstage/core-plugin-api": "^1.5.1", + "@backstage/plugin-catalog-react": "^1.7.0", + "@backstage/plugin-kubernetes": "^0.9.2", + "@backstage/theme": "^0.3.0", + "@material-ui/core": "^4.12.2", + "@material-ui/icons": "^4.9.1", + "@material-ui/lab": "4.0.0-alpha.61", + "react-use": "^17.2.4", + "yaml": "^2.3.1" + }, + "peerDependencies": { + "react": "^16.13.1 || ^17.0.0" + }, + "devDependencies": { + "@backstage/cli": "^0.22.7", + "@backstage/core-app-api": "^1.8.0", + "@backstage/dev-utils": "^1.0.15", + "@backstage/test-utils": "^1.3.1", + "@testing-library/jest-dom": "^5.10.1", + "@testing-library/react": "^12.1.3", + "@testing-library/user-event": "^14.0.0", + "@types/node": "*", + "cross-fetch": "^3.1.5", + "msw": "^1.0.0" + }, + "files": [ + "dist" + ] +} diff --git a/plugins/apache-spark/pi.yaml b/plugins/apache-spark/pi.yaml new file mode 100644 index 0000000..4e90932 --- /dev/null +++ b/plugins/apache-spark/pi.yaml @@ -0,0 +1,57 @@ +# +# Copyright 2017 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: "sparkoperator.k8s.io/v1beta2" +kind: SparkApplication +metadata: +# name: spark-pi + generateName: spark-pi + namespace: default +spec: + type: Python + pythonVersion: "3" + mode: cluster + image: "public.ecr.aws/m8u6z8z4/manabu-test:test-spark" + imagePullPolicy: Always + mainClass: org.apache.spark.examples.SparkPi + mainApplicationFile: "local:///opt/spark/examples/src/main/python/pi.ps" + sparkVersion: "3.1.1" + restartPolicy: + type: Never + volumes: + - name: "test-volume" + hostPath: + path: "/tmp" + type: Directory + driver: + cores: 1 + coreLimit: "1200m" + memory: "512m" + labels: + version: 3.1.1 + serviceAccount: spark + volumeMounts: + - name: "test-volume" + mountPath: "/tmp" + executor: + cores: 1 + instances: 1 + memory: "512m" + labels: + version: 3.1.1 + volumeMounts: + - name: "test-volume" + mountPath: "/tmp" + diff --git a/plugins/apache-spark/rbac.yaml b/plugins/apache-spark/rbac.yaml new file mode 100644 index 0000000..89ea433 --- /dev/null +++ b/plugins/apache-spark/rbac.yaml @@ -0,0 +1,35 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: spark + namespace: default +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + namespace: default + name: spark-role +rules: +- apiGroups: [""] + resources: ["pods"] + verbs: ["*"] +- apiGroups: [""] + resources: ["services"] + verbs: ["*"] +- apiGroups: [""] + resources: ["configmaps"] + verbs: ["*"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: spark-role-binding + namespace: default +subjects: +- kind: ServiceAccount + name: spark + namespace: default +roleRef: + kind: Role + name: spark-role + apiGroup: rbac.authorization.k8s.io diff --git a/plugins/apache-spark/src/api/index.ts b/plugins/apache-spark/src/api/index.ts new file mode 100644 index 0000000..eb32777 --- /dev/null +++ b/plugins/apache-spark/src/api/index.ts @@ -0,0 +1,103 @@ +import { createApiRef } from '@backstage/core-plugin-api'; +import { ApacheSpark, ApacheSparkList } from './model'; +import { KubernetesApi } from '@backstage/plugin-kubernetes'; + +export const apacheSparkApiRef = createApiRef({ + id: 'plugin.apachespark', +}); + +const API_VERSION = 'sparkoperator.k8s.io/v1beta2'; +const SPARK_APP_PLURAL = 'sparkapplications'; +const K8s_API_TIMEOUT = 'timeoutSeconds'; + +export interface ApacheSparkApi { + getSparkApps( + clusterName: string | undefined, + namespace: string | undefined, + labels: string | undefined, + ): Promise; + + getSparkApp( + clusterName: string | undefined, + namespace: string | undefined, + name: string, + ): Promise; +} + +export class ApacheSparkClient implements ApacheSparkApi { + private kubernetesApi: KubernetesApi; + constructor(kubernetesApi: KubernetesApi) { + this.kubernetesApi = kubernetesApi; + } + async getSparkApps( + clusterName: string | undefined, + namespace: string | undefined, + labels: string, + ): Promise { + const ns = namespace !== undefined ? namespace : 'default'; + const path = `/apis/${API_VERSION}/namespaces/${ns}/${SPARK_APP_PLURAL}`; + const query = new URLSearchParams({ + [K8s_API_TIMEOUT]: '30', + // labelSelector: labels, + }); + const resp = await this.kubernetesApi.proxy({ + clusterName: + clusterName !== undefined ? clusterName : await this.getFirstCluster(), + path: `${path}?${query.toString()}`, + }); + + if (!resp.ok) { + return Promise.reject( + `failed to fetch resources: ${resp.status}, ${ + resp.statusText + }, ${await resp.text()}`, + ); + } + const out = JSON.parse(await resp.text()); + this.removeManagedField(out); + return out; + } + + async getSparkApp( + clusterName: string | undefined, + namespace: string | undefined, + name: string, + ): Promise { + const ns = namespace !== undefined ? namespace : 'default'; + const path = `/apis/${API_VERSION}/namespaces/${ns}/${SPARK_APP_PLURAL}/${name}`; + const resp = await this.kubernetesApi.proxy({ + clusterName: + clusterName !== undefined ? clusterName : await this.getFirstCluster(), + path: `${path}`, + }); + if (!resp.ok) { + return Promise.reject( + `failed to fetch resources: ${resp.status}, ${ + resp.statusText + }, ${await resp.text()}`, + ); + } + const out = JSON.parse(await resp.text()); + this.removeManagedField(out); + return out; + } + + async getFirstCluster(): Promise { + const clusters = await this.kubernetesApi.getClusters(); + if (clusters.length > 0) { + return Promise.resolve(clusters[0].name); + } + return Promise.reject('no clusters found in configuration'); + } + + removeManagedField(spark: any) { + if (spark.metadata?.hasOwnProperty('managedFields')) { + delete spark.metadata.managedFields; + } + if (spark.items) { + for (const i of spark.items) { + this.removeManagedField(i); + } + } + } +} diff --git a/plugins/apache-spark/src/api/model.ts b/plugins/apache-spark/src/api/model.ts new file mode 100644 index 0000000..2c1c282 --- /dev/null +++ b/plugins/apache-spark/src/api/model.ts @@ -0,0 +1,84 @@ +export type Metadata = { + name: string; + namespace?: string; + labels: Record; + annotations: Record; + creationTimestamp: string; + managedFields?: any; +}; + +export type Spec = { + arguments: string[]; + batchScheduler: string; + driver: { + coreLimit: string; + coreRequest: string; + cores: number; + gpu: { + name: string; + quantity: number; + }; + labels: Record; + memory: string; + memoryOverhead: string; + podName: string; + schedulerName: string; + serviceAccount: string; + }; + executor: { + coreLimit: string; + coreRequest: string; + cores: number; + gpu: { + name: string; + quantity: number; + }; + instances: number; + labels: Record; + memory: string; + memoryOverhead: string; + schedulerName: string; + serviceAccount: string; + }; + image: string; + mainClass: string; + mode: string; + pythonVersion: string; + type: string; +}; + +export type Status = { + applicationState: { + errorMessage: string; + state: string; + }; + driverInfo: { + podName: string; + webUIAddress: string; + webUIIngressAddress: string; + webUIIngressName: string; + webUIPort: string; + webUIServiceName: string; + }; + executionAttempts: number; + executorState: Record; + lastSubmissionAttemptTime: string; + sparkApplicationId: string; + submissionAttempts: number; + submissionID: string; + terminationTime: string; +}; + +export type ApacheSpark = { + apiVersion: string; + kind: string; + metadata: Metadata; + spec: Spec; + status: Status; +}; + +export type ApacheSparkList = { + apiVersion: string; + kind: string; + items?: ApacheSpark[]; +}; diff --git a/plugins/apache-spark/src/components/ApacheSparkOverviewTable/ApacheSparkOverviewTable.tsx b/plugins/apache-spark/src/components/ApacheSparkOverviewTable/ApacheSparkOverviewTable.tsx new file mode 100644 index 0000000..d13aef6 --- /dev/null +++ b/plugins/apache-spark/src/components/ApacheSparkOverviewTable/ApacheSparkOverviewTable.tsx @@ -0,0 +1,148 @@ +import { + Progress, + StatusError, + StatusOK, + StatusPending, + StatusRunning, + Table, + TableColumn, +} from '@backstage/core-components'; +import { useEntity } from '@backstage/plugin-catalog-react'; +import { useApi } from '@backstage/core-plugin-api'; +import { apacheSparkApiRef } from '../../api'; +import React, { useEffect, useState } from 'react'; +import { getAnnotationValues } from '../utils'; +import useAsync from 'react-use/lib/useAsync'; +import { ApacheSpark, ApacheSparkList } from '../../api/model'; +import Alert from '@material-ui/lab/Alert'; +import { createStyles, Drawer, makeStyles, Theme } from '@material-ui/core'; +import { DrawerContent } from '../DetailedDrawer/DetailedDrawer'; + +type TableData = { + id: string; + name: string; + namespace: string; + applicationState?: string; + startedAt?: string; + finishedAt?: string; + raw: ApacheSpark; +}; + +const columns: TableColumn[] = [ + { + title: 'Name', + field: 'name', + }, + { title: 'Namespace', field: 'namespace', type: 'string' }, + { + title: 'Application State', + field: 'applicationState', + }, + { + title: 'StartTime', + field: 'startedAt', + type: 'datetime', + defaultSort: 'desc', + }, + { title: 'EndTime', field: 'finishedAt', type: 'datetime' }, +]; + +const useDrawerStyles = makeStyles((theme: Theme) => + createStyles({ + paper: { + width: '50%', + justifyContent: 'space-between', + padding: theme.spacing(2.5), + }, + }), +); + +export const ApacheSparkOverviewTable = () => { + // const { entity } = useEntity(); + const apiClient = useApi(apacheSparkApiRef); + const [columnData, setColumnData] = useState([] as TableData[]); + const [isOpen, toggleDrawer] = useState(false); + const [drawerData, setDrawerData] = useState({} as ApacheSpark); + const classes = useDrawerStyles(); + // const { ns, clusterName, labelSelector } = getAnnotationValues(entity); + + const { value, loading, error } = useAsync( + async (): Promise => { + return await apiClient.getSparkApps( + 'cnoe-packaging-2', + 'default', + undefined, + ); + }, + ); + + useEffect(() => { + const data = value?.items?.map(val => { + let state = {}; + switch (val.status?.applicationState.state) { + case 'RUNNING': + state = Running; + break; + case 'COMPLETED': + state = COMPLETED; + break; + case 'FAILED': + state = FAILED; + break; + default: + state = ( + + '${val.status.applicationState.state}' + + ); + break; + } + return { + id: `${val.metadata.namespace}/${val.metadata.name}`, + raw: val, + name: val.metadata.name, + namespace: val.metadata.namespace, + applicationState: state, + startedAt: val.metadata.creationTimestamp, + finishedAt: val.status?.terminationTime, + } as TableData; + }); + if (data && data.length > 0) { + setColumnData(data); + } + }, [value]); + if (loading) { + return ; + } else if (error) { + return {`${error}`}; + } + + return ( + <> + { + setDrawerData(rowData?.raw!); + toggleDrawer(true); + }} + columns={columns} + data={columnData} + /> + toggleDrawer(false)} + > + + + + ); +}; diff --git a/plugins/apache-spark/src/components/DetailedDrawer/DetailedDrawer.tsx b/plugins/apache-spark/src/components/DetailedDrawer/DetailedDrawer.tsx new file mode 100644 index 0000000..25e906e --- /dev/null +++ b/plugins/apache-spark/src/components/DetailedDrawer/DetailedDrawer.tsx @@ -0,0 +1,91 @@ +import { ApacheSpark } from '../../api/model'; +import { + Button, + createStyles, + IconButton, + makeStyles, + Theme, + Typography, +} from '@material-ui/core'; +import Close from '@material-ui/icons/Close'; +import React, { PropsWithChildren } from 'react'; +import { stringify } from 'yaml'; +import { CopyTextButton, TabbedLayout } from '@backstage/core-components'; +import { MemoryRouter, Route, Routes } from 'react-router-dom'; + +const useDrawerContentStyles = makeStyles((theme: Theme) => + createStyles({ + header: { + display: 'flex', + flexDirection: 'row', + justifyContent: 'space-between', + }, + icon: { + fontSize: 20, + }, + content: { + height: '80%', + backgroundColor: '#EEEEEE', + overflow: 'scroll', + display: 'flex', + flexDirection: 'row', + }, + secondaryAction: { + marginLeft: theme.spacing(2.5), + }, + }), +); + +export const DrawerContent = ({ + toggleDrawer, + apacheSpark, +}: { + toggleDrawer: (isOpen: boolean) => void; + apacheSpark: ApacheSpark; +}) => { + const classes = useDrawerContentStyles(); + const yamlString = stringify(apacheSpark); + return ( + + + <> +
+ {apacheSpark.metadata.name} + toggleDrawer(false)} + color="inherit" + > + + +
+
+ + 
{yamlString}
+
+
+ + +
+ + + +
logs!
+ + + ); +}; diff --git a/plugins/apache-spark/src/components/Overvew/Overview.tsx b/plugins/apache-spark/src/components/Overvew/Overview.tsx new file mode 100644 index 0000000..92d87f2 --- /dev/null +++ b/plugins/apache-spark/src/components/Overvew/Overview.tsx @@ -0,0 +1,14 @@ +import { Content, Header, HeaderLabel, Page } from '@backstage/core-components'; +import { ApacheSparkOverviewTable } from '../ApacheSparkOverviewTable/ApacheSparkOverviewTable'; +import React from 'react'; + +export const ApacheSparkOverviewPage = () => ( + +
+ +
+ + + + +); diff --git a/plugins/apache-spark/src/components/Overvew/index.ts b/plugins/apache-spark/src/components/Overvew/index.ts new file mode 100644 index 0000000..64d1896 --- /dev/null +++ b/plugins/apache-spark/src/components/Overvew/index.ts @@ -0,0 +1 @@ +export * from './Overview'; diff --git a/plugins/apache-spark/src/components/utils.ts b/plugins/apache-spark/src/components/utils.ts new file mode 100644 index 0000000..4c4a5f1 --- /dev/null +++ b/plugins/apache-spark/src/components/utils.ts @@ -0,0 +1,31 @@ +import { Entity } from '@backstage/catalog-model'; +import { + APACHE_SPARK_LABEL_SELECTOR_ANNOTATION, + CLUSTER_NAME_ANNOTATION, + K8S_LABEL_SELECTOR_ANNOTATION, + K8S_NAMESPACE_ANNOTATION, +} from '../plugin'; + +export type getAnnotationValuesOutput = { + ns: string; + clusterName?: string; + labelSelector?: string; +}; + +export function getAnnotationValues(entity: Entity): getAnnotationValuesOutput { + const ns = + entity.metadata.annotations?.[K8S_NAMESPACE_ANNOTATION] !== undefined + ? entity.metadata.annotations?.[K8S_NAMESPACE_ANNOTATION] + : 'default'; + const clusterName = entity.metadata.annotations?.[CLUSTER_NAME_ANNOTATION]; + const labelSelector = + entity.metadata?.annotations?.[APACHE_SPARK_LABEL_SELECTOR_ANNOTATION] !== + undefined + ? entity.metadata?.annotations?.[APACHE_SPARK_LABEL_SELECTOR_ANNOTATION] + : entity.metadata.annotations?.[K8S_LABEL_SELECTOR_ANNOTATION]; + return { + ns: ns, + clusterName: clusterName, + labelSelector: labelSelector, + }; +} diff --git a/plugins/apache-spark/src/index.ts b/plugins/apache-spark/src/index.ts new file mode 100644 index 0000000..7365fa6 --- /dev/null +++ b/plugins/apache-spark/src/index.ts @@ -0,0 +1 @@ +export { apacheSparkPlugin, ApacheSparkPage } from './plugin'; diff --git a/plugins/apache-spark/src/plugin.test.ts b/plugins/apache-spark/src/plugin.test.ts new file mode 100644 index 0000000..0df10dc --- /dev/null +++ b/plugins/apache-spark/src/plugin.test.ts @@ -0,0 +1,7 @@ +import { apacheSparkPlugin } from './plugin'; + +describe('apache-spark', () => { + it('should export plugin', () => { + expect(apacheSparkPlugin).toBeDefined(); + }); +}); diff --git a/plugins/apache-spark/src/plugin.ts b/plugins/apache-spark/src/plugin.ts new file mode 100644 index 0000000..171a5eb --- /dev/null +++ b/plugins/apache-spark/src/plugin.ts @@ -0,0 +1,42 @@ +import { + createApiFactory, + createPlugin, + createRoutableExtension, + discoveryApiRef, + fetchApiRef, +} from '@backstage/core-plugin-api'; + +import { rootRouteRef } from './routes'; +import { apacheSparkApiRef, ApacheSparkClient } from './api'; +import { kubernetesApiRef } from '@backstage/plugin-kubernetes'; + +export const APACHE_SPARK_LABEL_SELECTOR_ANNOTATION = + 'apache-spark/label-selector'; +export const CLUSTER_NAME_ANNOTATION = 'apache-spark/cluster-name'; +export const K8S_LABEL_SELECTOR_ANNOTATION = + 'backstage.io/kubernetes-label-selector'; +export const K8S_NAMESPACE_ANNOTATION = 'backstage.io/kubernetes-namespace'; +export const apacheSparkPlugin = createPlugin({ + id: 'apache-spark', + routes: { + root: rootRouteRef, + }, + apis: [ + createApiFactory({ + api: apacheSparkApiRef, + deps: { + kubernetesApi: kubernetesApiRef, + }, + factory: ({ kubernetesApi }) => new ApacheSparkClient(kubernetesApi), + }), + ], +}); + +export const ApacheSparkPage = apacheSparkPlugin.provide( + createRoutableExtension({ + name: 'ApacheSparkPage', + component: () => + import('./components/Overvew').then(m => m.ApacheSparkOverviewPage), + mountPoint: rootRouteRef, + }), +); diff --git a/plugins/apache-spark/src/routes.ts b/plugins/apache-spark/src/routes.ts new file mode 100644 index 0000000..f2230a1 --- /dev/null +++ b/plugins/apache-spark/src/routes.ts @@ -0,0 +1,5 @@ +import { createRouteRef } from '@backstage/core-plugin-api'; + +export const rootRouteRef = createRouteRef({ + id: 'apache-spark', +}); diff --git a/plugins/apache-spark/src/setupTests.ts b/plugins/apache-spark/src/setupTests.ts new file mode 100644 index 0000000..48c09b5 --- /dev/null +++ b/plugins/apache-spark/src/setupTests.ts @@ -0,0 +1,2 @@ +import '@testing-library/jest-dom'; +import 'cross-fetch/polyfill'; diff --git a/plugins/apache-spark/test/t b/plugins/apache-spark/test/t new file mode 100644 index 0000000..2d05bfb --- /dev/null +++ b/plugins/apache-spark/test/t @@ -0,0 +1,276 @@ + + + + + + + + PythonPi - Spark Jobs + + + +
+
+
+

+ Spark Jobs + + (?) + +

+
+
+
+
+
+
    +
  • + + User: + root +
    • +
  • + Total Uptime: + 55 min +
    • +
  • + Scheduling Mode: + FIFO +
    • + +
  • + Completed Jobs: + 1 +
    • + +
+
+ + + Event Timeline + + +

+ + Completed Jobs (1) +

+
+
+
+
+
+ + + + + + + + + + +
+
+ Page: +
    + + +
  • 1
    • + + +
+
+
+
+ + + + + + + + + + + + + +
+ + + Job Id ▾ + + + + + + Description + + + + + + Submitted + + + + + + Duration + + + + + Stages: Succeeded/Total + + + + Tasks (for all stages): Succeeded/Total + +
+ 0 + + reduce at /opt/spark/examples/src/main/python/pi.py:43 + reduce at /opt/spark/examples/src/main/python/pi.py:43 + + 2023/06/30 22:59:33 + 14 s + 1/1 + + + +
+ + 2/2 + + + + + +
+
+
+
+
+
+
+ + + + + + + + + +
+
+
+ Page: +
    + + +
  • 1
    • + + +
+
+
+ + + + + + + \ No newline at end of file diff --git a/plugins/apache-spark/test/t.tgz b/plugins/apache-spark/test/t.tgz new file mode 100644 index 0000000..d60cc24 Binary files /dev/null and b/plugins/apache-spark/test/t.tgz differ