From 0e5a38664c1db473c422f68f50bb3c76cb421c4c Mon Sep 17 00:00:00 2001 From: Manabu Mccloskey Date: Mon, 3 Jul 2023 17:24:10 -0700 Subject: [PATCH] wip --- plugins/apache-spark/.eslintrc.js | 3 + plugins/apache-spark/.prettierignore | 2 + plugins/apache-spark/README.md | 13 + plugins/apache-spark/dev/index.tsx | 12 + plugins/apache-spark/package.json | 55 ++++ plugins/apache-spark/pi.yaml | 57 ++++ plugins/apache-spark/rbac.yaml | 35 +++ plugins/apache-spark/src/api/index.ts | 103 +++++++ plugins/apache-spark/src/api/model.ts | 84 ++++++ .../ApacheSparkOverviewTable.tsx | 148 ++++++++++ .../DetailedDrawer/DetailedDrawer.tsx | 91 ++++++ .../src/components/Overvew/Overview.tsx | 14 + .../src/components/Overvew/index.ts | 1 + plugins/apache-spark/src/components/utils.ts | 31 ++ plugins/apache-spark/src/index.ts | 1 + plugins/apache-spark/src/plugin.test.ts | 7 + plugins/apache-spark/src/plugin.ts | 42 +++ plugins/apache-spark/src/routes.ts | 5 + plugins/apache-spark/src/setupTests.ts | 2 + plugins/apache-spark/test/t | 276 ++++++++++++++++++ plugins/apache-spark/test/t.tgz | Bin 0 -> 3005 bytes 21 files changed, 982 insertions(+) create mode 100644 plugins/apache-spark/.eslintrc.js create mode 100644 plugins/apache-spark/.prettierignore create mode 100644 plugins/apache-spark/README.md create mode 100644 plugins/apache-spark/dev/index.tsx create mode 100644 plugins/apache-spark/package.json create mode 100644 plugins/apache-spark/pi.yaml create mode 100644 plugins/apache-spark/rbac.yaml create mode 100644 plugins/apache-spark/src/api/index.ts create mode 100644 plugins/apache-spark/src/api/model.ts create mode 100644 plugins/apache-spark/src/components/ApacheSparkOverviewTable/ApacheSparkOverviewTable.tsx create mode 100644 plugins/apache-spark/src/components/DetailedDrawer/DetailedDrawer.tsx create mode 100644 plugins/apache-spark/src/components/Overvew/Overview.tsx create mode 100644 plugins/apache-spark/src/components/Overvew/index.ts create mode 100644 plugins/apache-spark/src/components/utils.ts create mode 100644 plugins/apache-spark/src/index.ts create mode 100644 plugins/apache-spark/src/plugin.test.ts create mode 100644 plugins/apache-spark/src/plugin.ts create mode 100644 plugins/apache-spark/src/routes.ts create mode 100644 plugins/apache-spark/src/setupTests.ts create mode 100644 plugins/apache-spark/test/t create mode 100644 plugins/apache-spark/test/t.tgz diff --git a/plugins/apache-spark/.eslintrc.js b/plugins/apache-spark/.eslintrc.js new file mode 100644 index 0000000..998aac2 --- /dev/null +++ b/plugins/apache-spark/.eslintrc.js @@ -0,0 +1,3 @@ +module.exports = require('@backstage/cli/config/eslint-factory')(__dirname, { + extends: ['prettier'], +}); diff --git a/plugins/apache-spark/.prettierignore b/plugins/apache-spark/.prettierignore new file mode 100644 index 0000000..5498e0f --- /dev/null +++ b/plugins/apache-spark/.prettierignore @@ -0,0 +1,2 @@ +build +coverage diff --git a/plugins/apache-spark/README.md b/plugins/apache-spark/README.md new file mode 100644 index 0000000..976aba2 --- /dev/null +++ b/plugins/apache-spark/README.md @@ -0,0 +1,13 @@ +# apache-spark + +Welcome to the apache-spark plugin! + +_This plugin was created through the Backstage CLI_ + +## Getting started + +Your plugin has been added to the example app in this repository, meaning you'll be able to access it by running `yarn start` in the root directory, and then navigating to [/apache-spark](http://localhost:3000/apache-spark). + +You can also serve the plugin in isolation by running `yarn start` in the plugin directory. +This method of serving the plugin provides quicker iteration speed and a faster startup and hot reloads. +It is only meant for local development, and the setup for it can be found inside the [/dev](./dev) directory. diff --git a/plugins/apache-spark/dev/index.tsx b/plugins/apache-spark/dev/index.tsx new file mode 100644 index 0000000..5f2b474 --- /dev/null +++ b/plugins/apache-spark/dev/index.tsx @@ -0,0 +1,12 @@ +import React from 'react'; +import { createDevApp } from '@backstage/dev-utils'; +import { apacheSparkPlugin, ApacheSparkPage } from '../src/plugin'; + +createDevApp() + .registerPlugin(apacheSparkPlugin) + .addPage({ + element: , + title: 'Root Page', + path: '/apache-spark' + }) + .render(); diff --git a/plugins/apache-spark/package.json b/plugins/apache-spark/package.json new file mode 100644 index 0000000..28ec3df --- /dev/null +++ b/plugins/apache-spark/package.json @@ -0,0 +1,55 @@ +{ + "name": "@internal/plugin-apache-spark", + "version": "0.1.0", + "main": "src/index.ts", + "types": "src/index.ts", + "license": "Apache-2.0", + "private": true, + "publishConfig": { + "access": "public", + "main": "dist/index.esm.js", + "types": "dist/index.d.ts" + }, + "backstage": { + "role": "frontend-plugin" + }, + "scripts": { + "start": "backstage-cli package start", + "build": "backstage-cli package build", + "lint": "backstage-cli package lint", + "test": "backstage-cli package test", + "clean": "backstage-cli package clean", + "prepack": "backstage-cli package prepack", + "postpack": "backstage-cli package postpack" + }, + "dependencies": { + "@backstage/core-components": "^0.13.1", + "@backstage/core-plugin-api": "^1.5.1", + "@backstage/plugin-catalog-react": "^1.7.0", + "@backstage/plugin-kubernetes": "^0.9.2", + "@backstage/theme": "^0.3.0", + "@material-ui/core": "^4.12.2", + "@material-ui/icons": "^4.9.1", + "@material-ui/lab": "4.0.0-alpha.61", + "react-use": "^17.2.4", + "yaml": "^2.3.1" + }, + "peerDependencies": { + "react": "^16.13.1 || ^17.0.0" + }, + "devDependencies": { + "@backstage/cli": "^0.22.7", + "@backstage/core-app-api": "^1.8.0", + "@backstage/dev-utils": "^1.0.15", + "@backstage/test-utils": "^1.3.1", + "@testing-library/jest-dom": "^5.10.1", + "@testing-library/react": "^12.1.3", + "@testing-library/user-event": "^14.0.0", + "@types/node": "*", + "cross-fetch": "^3.1.5", + "msw": "^1.0.0" + }, + "files": [ + "dist" + ] +} diff --git a/plugins/apache-spark/pi.yaml b/plugins/apache-spark/pi.yaml new file mode 100644 index 0000000..4e90932 --- /dev/null +++ b/plugins/apache-spark/pi.yaml @@ -0,0 +1,57 @@ +# +# Copyright 2017 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: "sparkoperator.k8s.io/v1beta2" +kind: SparkApplication +metadata: +# name: spark-pi + generateName: spark-pi + namespace: default +spec: + type: Python + pythonVersion: "3" + mode: cluster + image: "public.ecr.aws/m8u6z8z4/manabu-test:test-spark" + imagePullPolicy: Always + mainClass: org.apache.spark.examples.SparkPi + mainApplicationFile: "local:///opt/spark/examples/src/main/python/pi.ps" + sparkVersion: "3.1.1" + restartPolicy: + type: Never + volumes: + - name: "test-volume" + hostPath: + path: "/tmp" + type: Directory + driver: + cores: 1 + coreLimit: "1200m" + memory: "512m" + labels: + version: 3.1.1 + serviceAccount: spark + volumeMounts: + - name: "test-volume" + mountPath: "/tmp" + executor: + cores: 1 + instances: 1 + memory: "512m" + labels: + version: 3.1.1 + volumeMounts: + - name: "test-volume" + mountPath: "/tmp" + diff --git a/plugins/apache-spark/rbac.yaml b/plugins/apache-spark/rbac.yaml new file mode 100644 index 0000000..89ea433 --- /dev/null +++ b/plugins/apache-spark/rbac.yaml @@ -0,0 +1,35 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: spark + namespace: default +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + namespace: default + name: spark-role +rules: +- apiGroups: [""] + resources: ["pods"] + verbs: ["*"] +- apiGroups: [""] + resources: ["services"] + verbs: ["*"] +- apiGroups: [""] + resources: ["configmaps"] + verbs: ["*"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: spark-role-binding + namespace: default +subjects: +- kind: ServiceAccount + name: spark + namespace: default +roleRef: + kind: Role + name: spark-role + apiGroup: rbac.authorization.k8s.io diff --git a/plugins/apache-spark/src/api/index.ts b/plugins/apache-spark/src/api/index.ts new file mode 100644 index 0000000..eb32777 --- /dev/null +++ b/plugins/apache-spark/src/api/index.ts @@ -0,0 +1,103 @@ +import { createApiRef } from '@backstage/core-plugin-api'; +import { ApacheSpark, ApacheSparkList } from './model'; +import { KubernetesApi } from '@backstage/plugin-kubernetes'; + +export const apacheSparkApiRef = createApiRef({ + id: 'plugin.apachespark', +}); + +const API_VERSION = 'sparkoperator.k8s.io/v1beta2'; +const SPARK_APP_PLURAL = 'sparkapplications'; +const K8s_API_TIMEOUT = 'timeoutSeconds'; + +export interface ApacheSparkApi { + getSparkApps( + clusterName: string | undefined, + namespace: string | undefined, + labels: string | undefined, + ): Promise; + + getSparkApp( + clusterName: string | undefined, + namespace: string | undefined, + name: string, + ): Promise; +} + +export class ApacheSparkClient implements ApacheSparkApi { + private kubernetesApi: KubernetesApi; + constructor(kubernetesApi: KubernetesApi) { + this.kubernetesApi = kubernetesApi; + } + async getSparkApps( + clusterName: string | undefined, + namespace: string | undefined, + labels: string, + ): Promise { + const ns = namespace !== undefined ? namespace : 'default'; + const path = `/apis/${API_VERSION}/namespaces/${ns}/${SPARK_APP_PLURAL}`; + const query = new URLSearchParams({ + [K8s_API_TIMEOUT]: '30', + // labelSelector: labels, + }); + const resp = await this.kubernetesApi.proxy({ + clusterName: + clusterName !== undefined ? clusterName : await this.getFirstCluster(), + path: `${path}?${query.toString()}`, + }); + + if (!resp.ok) { + return Promise.reject( + `failed to fetch resources: ${resp.status}, ${ + resp.statusText + }, ${await resp.text()}`, + ); + } + const out = JSON.parse(await resp.text()); + this.removeManagedField(out); + return out; + } + + async getSparkApp( + clusterName: string | undefined, + namespace: string | undefined, + name: string, + ): Promise { + const ns = namespace !== undefined ? namespace : 'default'; + const path = `/apis/${API_VERSION}/namespaces/${ns}/${SPARK_APP_PLURAL}/${name}`; + const resp = await this.kubernetesApi.proxy({ + clusterName: + clusterName !== undefined ? clusterName : await this.getFirstCluster(), + path: `${path}`, + }); + if (!resp.ok) { + return Promise.reject( + `failed to fetch resources: ${resp.status}, ${ + resp.statusText + }, ${await resp.text()}`, + ); + } + const out = JSON.parse(await resp.text()); + this.removeManagedField(out); + return out; + } + + async getFirstCluster(): Promise { + const clusters = await this.kubernetesApi.getClusters(); + if (clusters.length > 0) { + return Promise.resolve(clusters[0].name); + } + return Promise.reject('no clusters found in configuration'); + } + + removeManagedField(spark: any) { + if (spark.metadata?.hasOwnProperty('managedFields')) { + delete spark.metadata.managedFields; + } + if (spark.items) { + for (const i of spark.items) { + this.removeManagedField(i); + } + } + } +} diff --git a/plugins/apache-spark/src/api/model.ts b/plugins/apache-spark/src/api/model.ts new file mode 100644 index 0000000..2c1c282 --- /dev/null +++ b/plugins/apache-spark/src/api/model.ts @@ -0,0 +1,84 @@ +export type Metadata = { + name: string; + namespace?: string; + labels: Record; + annotations: Record; + creationTimestamp: string; + managedFields?: any; +}; + +export type Spec = { + arguments: string[]; + batchScheduler: string; + driver: { + coreLimit: string; + coreRequest: string; + cores: number; + gpu: { + name: string; + quantity: number; + }; + labels: Record; + memory: string; + memoryOverhead: string; + podName: string; + schedulerName: string; + serviceAccount: string; + }; + executor: { + coreLimit: string; + coreRequest: string; + cores: number; + gpu: { + name: string; + quantity: number; + }; + instances: number; + labels: Record; + memory: string; + memoryOverhead: string; + schedulerName: string; + serviceAccount: string; + }; + image: string; + mainClass: string; + mode: string; + pythonVersion: string; + type: string; +}; + +export type Status = { + applicationState: { + errorMessage: string; + state: string; + }; + driverInfo: { + podName: string; + webUIAddress: string; + webUIIngressAddress: string; + webUIIngressName: string; + webUIPort: string; + webUIServiceName: string; + }; + executionAttempts: number; + executorState: Record; + lastSubmissionAttemptTime: string; + sparkApplicationId: string; + submissionAttempts: number; + submissionID: string; + terminationTime: string; +}; + +export type ApacheSpark = { + apiVersion: string; + kind: string; + metadata: Metadata; + spec: Spec; + status: Status; +}; + +export type ApacheSparkList = { + apiVersion: string; + kind: string; + items?: ApacheSpark[]; +}; diff --git a/plugins/apache-spark/src/components/ApacheSparkOverviewTable/ApacheSparkOverviewTable.tsx b/plugins/apache-spark/src/components/ApacheSparkOverviewTable/ApacheSparkOverviewTable.tsx new file mode 100644 index 0000000..d13aef6 --- /dev/null +++ b/plugins/apache-spark/src/components/ApacheSparkOverviewTable/ApacheSparkOverviewTable.tsx @@ -0,0 +1,148 @@ +import { + Progress, + StatusError, + StatusOK, + StatusPending, + StatusRunning, + Table, + TableColumn, +} from '@backstage/core-components'; +import { useEntity } from '@backstage/plugin-catalog-react'; +import { useApi } from '@backstage/core-plugin-api'; +import { apacheSparkApiRef } from '../../api'; +import React, { useEffect, useState } from 'react'; +import { getAnnotationValues } from '../utils'; +import useAsync from 'react-use/lib/useAsync'; +import { ApacheSpark, ApacheSparkList } from '../../api/model'; +import Alert from '@material-ui/lab/Alert'; +import { createStyles, Drawer, makeStyles, Theme } from '@material-ui/core'; +import { DrawerContent } from '../DetailedDrawer/DetailedDrawer'; + +type TableData = { + id: string; + name: string; + namespace: string; + applicationState?: string; + startedAt?: string; + finishedAt?: string; + raw: ApacheSpark; +}; + +const columns: TableColumn[] = [ + { + title: 'Name', + field: 'name', + }, + { title: 'Namespace', field: 'namespace', type: 'string' }, + { + title: 'Application State', + field: 'applicationState', + }, + { + title: 'StartTime', + field: 'startedAt', + type: 'datetime', + defaultSort: 'desc', + }, + { title: 'EndTime', field: 'finishedAt', type: 'datetime' }, +]; + +const useDrawerStyles = makeStyles((theme: Theme) => + createStyles({ + paper: { + width: '50%', + justifyContent: 'space-between', + padding: theme.spacing(2.5), + }, + }), +); + +export const ApacheSparkOverviewTable = () => { + // const { entity } = useEntity(); + const apiClient = useApi(apacheSparkApiRef); + const [columnData, setColumnData] = useState([] as TableData[]); + const [isOpen, toggleDrawer] = useState(false); + const [drawerData, setDrawerData] = useState({} as ApacheSpark); + const classes = useDrawerStyles(); + // const { ns, clusterName, labelSelector } = getAnnotationValues(entity); + + const { value, loading, error } = useAsync( + async (): Promise => { + return await apiClient.getSparkApps( + 'cnoe-packaging-2', + 'default', + undefined, + ); + }, + ); + + useEffect(() => { + const data = value?.items?.map(val => { + let state = {}; + switch (val.status?.applicationState.state) { + case 'RUNNING': + state = Running; + break; + case 'COMPLETED': + state = COMPLETED; + break; + case 'FAILED': + state = FAILED; + break; + default: + state = ( + + '${val.status.applicationState.state}' + + ); + break; + } + return { + id: `${val.metadata.namespace}/${val.metadata.name}`, + raw: val, + name: val.metadata.name, + namespace: val.metadata.namespace, + applicationState: state, + startedAt: val.metadata.creationTimestamp, + finishedAt: val.status?.terminationTime, + } as TableData; + }); + if (data && data.length > 0) { + setColumnData(data); + } + }, [value]); + if (loading) { + return ; + } else if (error) { + return {`${error}`}; + } + + return ( + <> + { + setDrawerData(rowData?.raw!); + toggleDrawer(true); + }} + columns={columns} + data={columnData} + /> + toggleDrawer(false)} + > + + + + ); +}; diff --git a/plugins/apache-spark/src/components/DetailedDrawer/DetailedDrawer.tsx b/plugins/apache-spark/src/components/DetailedDrawer/DetailedDrawer.tsx new file mode 100644 index 0000000..25e906e --- /dev/null +++ b/plugins/apache-spark/src/components/DetailedDrawer/DetailedDrawer.tsx @@ -0,0 +1,91 @@ +import { ApacheSpark } from '../../api/model'; +import { + Button, + createStyles, + IconButton, + makeStyles, + Theme, + Typography, +} from '@material-ui/core'; +import Close from '@material-ui/icons/Close'; +import React, { PropsWithChildren } from 'react'; +import { stringify } from 'yaml'; +import { CopyTextButton, TabbedLayout } from '@backstage/core-components'; +import { MemoryRouter, Route, Routes } from 'react-router-dom'; + +const useDrawerContentStyles = makeStyles((theme: Theme) => + createStyles({ + header: { + display: 'flex', + flexDirection: 'row', + justifyContent: 'space-between', + }, + icon: { + fontSize: 20, + }, + content: { + height: '80%', + backgroundColor: '#EEEEEE', + overflow: 'scroll', + display: 'flex', + flexDirection: 'row', + }, + secondaryAction: { + marginLeft: theme.spacing(2.5), + }, + }), +); + +export const DrawerContent = ({ + toggleDrawer, + apacheSpark, +}: { + toggleDrawer: (isOpen: boolean) => void; + apacheSpark: ApacheSpark; +}) => { + const classes = useDrawerContentStyles(); + const yamlString = stringify(apacheSpark); + return ( + + + <> +
+ {apacheSpark.metadata.name} + toggleDrawer(false)} + color="inherit" + > + + +
+
+ + 
{yamlString}
+
+
+ + +
+ + + +
logs!
+ + + ); +}; diff --git a/plugins/apache-spark/src/components/Overvew/Overview.tsx b/plugins/apache-spark/src/components/Overvew/Overview.tsx new file mode 100644 index 0000000..92d87f2 --- /dev/null +++ b/plugins/apache-spark/src/components/Overvew/Overview.tsx @@ -0,0 +1,14 @@ +import { Content, Header, HeaderLabel, Page } from '@backstage/core-components'; +import { ApacheSparkOverviewTable } from '../ApacheSparkOverviewTable/ApacheSparkOverviewTable'; +import React from 'react'; + +export const ApacheSparkOverviewPage = () => ( + +
+ +
+ + + + +); diff --git a/plugins/apache-spark/src/components/Overvew/index.ts b/plugins/apache-spark/src/components/Overvew/index.ts new file mode 100644 index 0000000..64d1896 --- /dev/null +++ b/plugins/apache-spark/src/components/Overvew/index.ts @@ -0,0 +1 @@ +export * from './Overview'; diff --git a/plugins/apache-spark/src/components/utils.ts b/plugins/apache-spark/src/components/utils.ts new file mode 100644 index 0000000..4c4a5f1 --- /dev/null +++ b/plugins/apache-spark/src/components/utils.ts @@ -0,0 +1,31 @@ +import { Entity } from '@backstage/catalog-model'; +import { + APACHE_SPARK_LABEL_SELECTOR_ANNOTATION, + CLUSTER_NAME_ANNOTATION, + K8S_LABEL_SELECTOR_ANNOTATION, + K8S_NAMESPACE_ANNOTATION, +} from '../plugin'; + +export type getAnnotationValuesOutput = { + ns: string; + clusterName?: string; + labelSelector?: string; +}; + +export function getAnnotationValues(entity: Entity): getAnnotationValuesOutput { + const ns = + entity.metadata.annotations?.[K8S_NAMESPACE_ANNOTATION] !== undefined + ? entity.metadata.annotations?.[K8S_NAMESPACE_ANNOTATION] + : 'default'; + const clusterName = entity.metadata.annotations?.[CLUSTER_NAME_ANNOTATION]; + const labelSelector = + entity.metadata?.annotations?.[APACHE_SPARK_LABEL_SELECTOR_ANNOTATION] !== + undefined + ? entity.metadata?.annotations?.[APACHE_SPARK_LABEL_SELECTOR_ANNOTATION] + : entity.metadata.annotations?.[K8S_LABEL_SELECTOR_ANNOTATION]; + return { + ns: ns, + clusterName: clusterName, + labelSelector: labelSelector, + }; +} diff --git a/plugins/apache-spark/src/index.ts b/plugins/apache-spark/src/index.ts new file mode 100644 index 0000000..7365fa6 --- /dev/null +++ b/plugins/apache-spark/src/index.ts @@ -0,0 +1 @@ +export { apacheSparkPlugin, ApacheSparkPage } from './plugin'; diff --git a/plugins/apache-spark/src/plugin.test.ts b/plugins/apache-spark/src/plugin.test.ts new file mode 100644 index 0000000..0df10dc --- /dev/null +++ b/plugins/apache-spark/src/plugin.test.ts @@ -0,0 +1,7 @@ +import { apacheSparkPlugin } from './plugin'; + +describe('apache-spark', () => { + it('should export plugin', () => { + expect(apacheSparkPlugin).toBeDefined(); + }); +}); diff --git a/plugins/apache-spark/src/plugin.ts b/plugins/apache-spark/src/plugin.ts new file mode 100644 index 0000000..171a5eb --- /dev/null +++ b/plugins/apache-spark/src/plugin.ts @@ -0,0 +1,42 @@ +import { + createApiFactory, + createPlugin, + createRoutableExtension, + discoveryApiRef, + fetchApiRef, +} from '@backstage/core-plugin-api'; + +import { rootRouteRef } from './routes'; +import { apacheSparkApiRef, ApacheSparkClient } from './api'; +import { kubernetesApiRef } from '@backstage/plugin-kubernetes'; + +export const APACHE_SPARK_LABEL_SELECTOR_ANNOTATION = + 'apache-spark/label-selector'; +export const CLUSTER_NAME_ANNOTATION = 'apache-spark/cluster-name'; +export const K8S_LABEL_SELECTOR_ANNOTATION = + 'backstage.io/kubernetes-label-selector'; +export const K8S_NAMESPACE_ANNOTATION = 'backstage.io/kubernetes-namespace'; +export const apacheSparkPlugin = createPlugin({ + id: 'apache-spark', + routes: { + root: rootRouteRef, + }, + apis: [ + createApiFactory({ + api: apacheSparkApiRef, + deps: { + kubernetesApi: kubernetesApiRef, + }, + factory: ({ kubernetesApi }) => new ApacheSparkClient(kubernetesApi), + }), + ], +}); + +export const ApacheSparkPage = apacheSparkPlugin.provide( + createRoutableExtension({ + name: 'ApacheSparkPage', + component: () => + import('./components/Overvew').then(m => m.ApacheSparkOverviewPage), + mountPoint: rootRouteRef, + }), +); diff --git a/plugins/apache-spark/src/routes.ts b/plugins/apache-spark/src/routes.ts new file mode 100644 index 0000000..f2230a1 --- /dev/null +++ b/plugins/apache-spark/src/routes.ts @@ -0,0 +1,5 @@ +import { createRouteRef } from '@backstage/core-plugin-api'; + +export const rootRouteRef = createRouteRef({ + id: 'apache-spark', +}); diff --git a/plugins/apache-spark/src/setupTests.ts b/plugins/apache-spark/src/setupTests.ts new file mode 100644 index 0000000..48c09b5 --- /dev/null +++ b/plugins/apache-spark/src/setupTests.ts @@ -0,0 +1,2 @@ +import '@testing-library/jest-dom'; +import 'cross-fetch/polyfill'; diff --git a/plugins/apache-spark/test/t b/plugins/apache-spark/test/t new file mode 100644 index 0000000..2d05bfb --- /dev/null +++ b/plugins/apache-spark/test/t @@ -0,0 +1,276 @@ + + + + + + + + PythonPi - Spark Jobs + + + +
+
+
+

+ Spark Jobs + + (?) + +

+
+
+
+
+
+
    +
  • + + User: + root +
    • +
  • + Total Uptime: + 55 min +
    • +
  • + Scheduling Mode: + FIFO +
    • + +
  • + Completed Jobs: + 1 +
    • + +
+
+ + + Event Timeline + + +

+ + Completed Jobs (1) +

+
+
+
+
+
+ + + + + + + + + + +
+
+ Page: +
    + + +
  • 1
    • + + +
+
+
+
+ + + + + + + + + + + + + +
+ + + Job Id ▾ + + + + + + Description + + + + + + Submitted + + + + + + Duration + + + + + Stages: Succeeded/Total + + + + Tasks (for all stages): Succeeded/Total + +
+ 0 + + reduce at /opt/spark/examples/src/main/python/pi.py:43 + reduce at /opt/spark/examples/src/main/python/pi.py:43 + + 2023/06/30 22:59:33 + 14 s + 1/1 + + + +
+ + 2/2 + + + + + +
+
+
+
+
+
+
+ + + + + + + + + +
+
+
+ Page: +
    + + +
  • 1
    • + + +
+
+
+ + + + + + + \ No newline at end of file diff --git a/plugins/apache-spark/test/t.tgz b/plugins/apache-spark/test/t.tgz new file mode 100644 index 0000000000000000000000000000000000000000..d60cc24390cf293fcfafa417798f43a6f9a51604 GIT binary patch literal 3005 zcmV;u3qtfCiwFP!00000|Lj}cbK5o&fA?Rp$4n!8YKoF&JE<(GlP1nhGQDeBCl7Z` zA3$(s?xGrCzi&jr`WwX3gFbRfvzmX5})_6(sj2x1}?>Sd^`k>UbIAh6d=`mqVbOYD$=^0>W@PHAEQYA3!2oI!a$N zR57Ax0!l|he}C+$_sH#GNwI+(hydjkzuYblBDB=@^PS)14hz9K=XcoSTmorjgP_y0P70jj8GaO$kXSMKhHeZb0<*ymaYj-w28Bgu%-wW5Bws$Yw z>s{z)JY;x(oD0uGKccIJV49}tei+}QMYT82y|82=x<_ughZ+~+hdoQv?})-TM90Hp`wva;7YnI_Oal!B{{jv$AHhT<+$T@ACzEvR#^E^Sk&)TJ zXvn0gX1IEDtz*I8P;lYGOorFOR|_xiETg~I4F3yXgx3u^oNORqM5(32$;KxV@;@$a zQ^G?x3CjjXqgaC<6%UE;9y>K5fJI0dJTT2wcs6%YY{~ZWf@`f zF%E=8lcI0w1cEV=DlD6cj8HF~M>&b-RWFg0xF+?Jf>j(k-MKYkxOCQpRjrKpW?#Lv z42_-9h-j;Wy`5@m3*g$VxawrAdSa*oA#~SkQzd5n62a z=^=qbzM)d^1iAK{)p`B)pg?Xh$h44Veb3A1hXnBA$M4PZ{`h?ll$WtvI2_#S(pg9w z0YjGyj$mS>>oXC2=tnT6rQi|Rc)WDVXzi4w4NaGeD@jtPiscJ0pVcSU0M(jF(v;D_ zdQN!vYAA9H`NvkASa_T?Ut6A%YeG3ncg->y?u_|TT-^}q1Hs(sQC~yx4D6xO#y?F1 z!d${=#OL4(twnMMS25M-+Et%JF7g?O4azkWS96GI7$QIG#a64KdKfZBQ$^L;Kq645 zP6H7-OUnuS3(^jd#bKg|k0~PoCVAmRq)xf$&)`z{Kow|7qX;F2;9mpbHYNyu2#kI} z2t>y9{s9OHifr(idW&2Ak}}*s7{hZ$gAX7$5KBu7P>4YJTF4|{b>fRmgN_k$9}8j) zny3#7C|5K@pnB}AzBQJAuvZvFEm&;?@$7e}7yq*bwC%8tt&A$|X53mw$Lbhe?R7S6 zzGdDil=I;YB!$o|K@5=&%^{L+g(eK%rRIROgY?M>B$R&+@rxkFFk_TQ@DCB<9EKUZ zeD(5&8_2gg*#ssKNy<=T=&CG9h`b)kf6M&(Q8n6GmB^l#O?F?@g7-2|^KaTSq#Xd8 zt~NKNmRBpz91VX~)hjCex1Hb;Da94oUpHVvS?OASaOZw5w(dnNt`uCw$j!b}W^YNO z5yC@2lg-*E+h0XLxD=FQXf`s#wI0I@Ng}&t11T6Am#WD3^f@O!L-@B4iLn7gE>j4J zFuzwZA~RDz3XuTip2+lg*xA&|KHnhM@U~DwrRrkQ3i<7}G-fiW>28W#Z@%`QZjg|I zxhdhuoK7Iw#mO~-xxL7_pgc9+%$`hw7=sVK*qXJo?X8hknnH;mb0u~|UCU!NUP|-J zs}+qx<0aXUytP6VN^AFVN}_{=rC_CO5&!^db&nxWcfFA z$S^`<%o2&&hnooUGP*N0$4CcaG-=1uIXOwU&Z-bdVxa5YkkfWa7-Hz=^c>hO9Ww*2 zi@yccqZZMr?ULK2b7UW`^M}>KGaO28HVN7;omok6`os)+X80{U(|aQ>iHX<@Bb!a` z8rrcP^i$5V0dC=`jqzhf{dSdpJM6Ba+3mX)Fh5j%?%F-wwRS2C0z|WHm%_itrMoEk z?q0q%p=>vs+{zjYF|c*wWE5!yQ@hV1VrqDZ;J9FbfcfhWDOG_4~Y* z_TIxSF}a9(?%{3a? z&|;L?Q~G4mq1Ld)RflA*pZiqm)6?nf1y&alSxHxM7$Y6@>iglY`0vm^yA z7GB=I3$Dp%2HE8cdzZ*x}0R_47zaiboWl8EyD_1x|{R6P&#Td1^Nz2PpF>i`$l z4$-w4r?aZ7Ii5&$UdyRUIi8Y%E#wq+oT`jd2-!$g?tJfF3g7#dhRZr_H6)7^>T^*y zK8)C*uYo$*jw#b)P@T4_*lkS3O{It=DrJ+YhJ8)F`aZGyu!G<(Qq41RIi_=ySftGM zbAkNJg8;ckc?PLa)R@AY_)4%$<5^x$o=+!}ubRicBkyP@;2{xOcaQ@JU7tv(MXC?3 z1=bt|Oed4^@I~CkaueinRQtQ#`T%479ueWK{)A^({|Lpm0>I(Tm`>)c%pSQHo zuz&|rU1%cutHAJQFEI2zr0aYj_34kOI^=3^y0HKGZuQ>)0096052DRd+cy9Jx%%P5 literal 0 HcmV?d00001