DevFW-CICD/argocd-helm
16
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

add megalinter config. Fix my rubbish code to appease the linter

Browse source 
Signed-off-by: Tim Collins <tim@thecollins.team>
This commit is contained in:
Tim Collins 2025-01-24 09:37:23 +00:00
parent 5835ac4e5c
commit c304c37c51
No known key found for this signature in database
GPG key ID: 09F625E00301ED18
8 changed files with 182 additions and 90 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

21
scripts/release-analysis/.mega-linter.yaml Normal file
View file

@ -0,0 +1,21 @@
# Configuration file for MegaLinter
# Run megalinter locally with: `docker run --rm -v /var/run/docker.sock:/var/run/docker.sock:rw -v $(pwd):/tmp/lint:rw oxsecurity/megalinter:v8`
ENABLE:
- DOCKERFILE
- PYTHON
DISABLE_LINTERS:
- PYTHON_FLAKE8
- PYTHON_PYRIGHT
- PYTHON_RUFF
- PYTHON_BANDIT
CLEAR_REPORT_FOLDER: true
PYTHON_PYLINT_PRE_COMMANDS:
- command: pip install -r /tmp/lint/requirements.txt
venv: pylint
REPORT_OUTPUT_FOLDER: none
# You might want to enable this locally to fix some stuff without guessing what you need to change. Check before committing to git.
#APPLY_FIXES: all

2
scripts/release-analysis/Dockerfile
View file

@ -5,4 +5,4 @@ RUN pip install --no-cache-dir -r requirements.txt
COPY main.py fetch_releases.py fetch_helmet_releases.py merge_csvs.py plot_graph.py ./ COPY main.py fetch_releases.py fetch_helmet_releases.py merge_csvs.py plot_graph.py ./
RUN chmod +x main.py fetch_releases.py fetch_helmet_releases.py merge_csvs.py plot_graph.py RUN chmod +x main.py fetch_releases.py fetch_helmet_releases.py merge_csvs.py plot_graph.py
CMD [ "python", "./main.py" ] CMD [ "python", "-u", "./main.py" ]

6
scripts/release-analysis/README.md
View file

@ -19,3 +19,9 @@ docker run --rm -e GITHUB_TOKEN=$GITHUB_TOKEN -v ${PWD}:/app team-helm-analysis
``` ```
You should get 3 csvs and 4 graphs once this completes. It takes around 5 mins to run. You should get 3 csvs and 4 graphs once this completes. It takes around 5 mins to run.
## Linting
I ran megalinter against this just to give some confidence that it's not completely broken. It's not perfect, but it's something.
Run megalinter locally against this directory with: `docker run --rm -v /var/run/docker.sock:/var/run/docker.sock:rw -v $(pwd):/tmp/lint:rw oxsecurity/megalinter:v8`

48
scripts/release-analysis/fetch_helmet_releases.py
View file

@ -1,36 +1,40 @@
import requests
import csv import csv
from datetime import datetime
import os import os
from datetime import datetime
import requests
# GitHub repository URL # GitHub repository URL
repo_url = "https://api.github.com/repos/argoproj/argo-helm/releases" repo_url = "https://api.github.com/repos/argoproj/argo-helm/releases"
# Get the GitHub token from environment variables # Get the GitHub token from environment variables
github_token = os.getenv('GITHUB_TOKEN') github_token = os.getenv("GITHUB_TOKEN")
if not github_token: if not github_token:
raise ValueError("GITHUB_TOKEN environment variable is not set") raise ValueError("GITHUB_TOKEN environment variable is not set")
# Function to fetch all releases with pagination # Function to fetch all releases with pagination
def fetch_all_releases(url): def fetch_all_releases(url):
releases = [] releases = []
headers = {'Authorization': f'token {github_token}'} headers = {"Authorization": f"token {github_token}"}
while url: while url:
response = requests.get(url, headers=headers) response = requests.get(url, headers=headers)
response.raise_for_status() response.raise_for_status()
releases.extend(response.json()) releases.extend(response.json())
url = response.links.get('next', {}).get('url') url = response.links.get("next", {}).get("url")
return releases return releases
# Function to get the content of Chart.yaml in a release # Function to get the content of Chart.yaml in a release
def get_chart_yaml(repo, tag, chart_path): def get_chart_yaml(repo, tag, chart_path):
url = f"https://raw.githubusercontent.com/{repo}/refs/tags/{tag}/charts/{chart_path}/Chart.yaml" url = f"https://raw.githubusercontent.com/{repo}/refs/tags/{tag}/charts/{chart_path}/Chart.yaml"
headers = {'Authorization': f'token {github_token}'} headers = {"Authorization": f"token {github_token}"}
response = requests.get(url, headers=headers) response = requests.get(url, headers=headers)
if response.status_code == 200: if response.status_code == 200:
return response.text return response.text
return None return None
# Function to extract appVersion from Chart.yaml content # Function to extract appVersion from Chart.yaml content
def extract_app_version(chart_yaml): def extract_app_version(chart_yaml):
for line in chart_yaml.splitlines(): for line in chart_yaml.splitlines():
@ -38,41 +42,49 @@ def extract_app_version(chart_yaml):
return line.split(":")[1].strip() return line.split(":")[1].strip()
return None return None
# Function to fetch releases and write to a CSV file # Function to fetch releases and write to a CSV file
def fetch_and_write_helmet_releases(csv_file): def fetch_and_write_helmet_releases(csv_file):
# Fetch all releases # Fetch all releases
releases = fetch_all_releases(repo_url) releases = fetch_all_releases(repo_url)
# Write the release data to the CSV file # Write the release data to the CSV file
with open(csv_file, mode='w', newline='') as file: with open(csv_file, mode="w", newline="") as file:
writer = csv.writer(file, quoting=csv.QUOTE_NONE, escapechar='\\') writer = csv.writer(file, quoting=csv.QUOTE_NONE, escapechar="\\")
writer.writerow(["Release Name", "Release Date", "Release Time", "App Version"]) writer.writerow(["Release Name", "Release Date", "Release Time", "App Version"])
for release in releases: for release in releases:
tag_name = release['tag_name'] tag_name = release["tag_name"]
published_at = release['published_at'] published_at = release["published_at"]
release_date = datetime.strptime(published_at, "%Y-%m-%dT%H:%M:%SZ").date() release_date = datetime.strptime(published_at, "%Y-%m-%dT%H:%M:%SZ").date()
release_time = datetime.strptime(published_at, "%Y-%m-%dT%H:%M:%SZ").time() release_time = datetime.strptime(published_at, "%Y-%m-%dT%H:%M:%SZ").time()
# Extract chart path from the release name # Extract chart path from the release name
chart_path = '-'.join(tag_name.split('-')[:-1]) chart_path = "-".join(tag_name.split("-")[:-1])
current_chart_yaml = get_chart_yaml("argoproj/argo-helm", tag_name, chart_path) current_chart_yaml = get_chart_yaml(
"argoproj/argo-helm", tag_name, chart_path
)
if current_chart_yaml: if current_chart_yaml:
current_app_version = extract_app_version(current_chart_yaml) current_app_version = extract_app_version(current_chart_yaml)
writer.writerow([tag_name, release_date, release_time, current_app_version]) writer.writerow(
[tag_name, release_date, release_time, current_app_version]
)
# Read the CSV file, remove any instances of `\"`, and write back the cleaned content # Read the CSV file, remove any instances of `\"`, and write back the cleaned content
with open(csv_file, mode='r') as file: with open(csv_file, mode="r") as file:
content = file.read() content = file.read()
cleaned_content = content.replace('\\"', '') cleaned_content = content.replace('\\"', "")
with open(csv_file, mode='w', newline='') as file: with open(csv_file, mode="w", newline="") as file:
file.write(cleaned_content) file.write(cleaned_content)
print(f"Release data has been written to {csv_file} and cleaned of any instances of \\\"") print(
f'Release data has been written to {csv_file} and cleaned of any instances of \\"'
)
# Example usage # Example usage
if __name__ == "__main__": if __name__ == "__main__":
fetch_and_write_helmet_releases('argo_helm_releases.csv') fetch_and_write_helmet_releases("argo_helm_releases.csv")

30
scripts/release-analysis/fetch_releases.py
View file

@ -1,7 +1,8 @@
import requests
import csv import csv
from datetime import datetime
import os import os
from datetime import datetime
import requests
# List of GitHub repository URLs we care about # List of GitHub repository URLs we care about
repos = [ repos = [
@ -12,38 +13,45 @@ repos = [
] ]
# Get the GitHub token from environment variables # Get the GitHub token from environment variables
github_token = os.getenv('GITHUB_TOKEN') github_token = os.getenv("GITHUB_TOKEN")
if not github_token: if not github_token:
raise ValueError("GITHUB_TOKEN environment variable is not set") raise ValueError("GITHUB_TOKEN environment variable is not set")
# Fetch all releases with pagination # Fetch all releases with pagination
def fetch_all_releases(url): def fetch_all_releases(url):
releases = [] releases = []
headers = {'Authorization': f'token {github_token}'} headers = {"Authorization": f"token {github_token}"}
while url: while url:
response = requests.get(url, headers=headers) response = requests.get(url, headers=headers)
response.raise_for_status() response.raise_for_status()
releases.extend(response.json()) releases.extend(response.json())
url = response.links.get('next', {}).get('url') url = response.links.get("next", {}).get("url")
return releases return releases
# Fetch releases and write to a CSV file # Fetch releases and write to a CSV file
def fetch_and_write_releases(csv_file): def fetch_and_write_releases(csv_file):
with open(csv_file, mode='w', newline='') as file: with open(csv_file, mode="w", newline="") as file:
writer = csv.writer(file) writer = csv.writer(file)
writer.writerow(["Repository", "Release Tag", "Release Date", "Release Time"]) writer.writerow(["Repository", "Release Tag", "Release Date", "Release Time"])
for repo_name, repo_url in repos: for repo_name, repo_url in repos:
releases = fetch_all_releases(repo_url) releases = fetch_all_releases(repo_url)
for release in releases: for release in releases:
tag_name = release['tag_name'] tag_name = release["tag_name"]
published_at = release['published_at'] published_at = release["published_at"]
release_date = datetime.strptime(published_at, "%Y-%m-%dT%H:%M:%SZ").date() release_date = datetime.strptime(
release_time = datetime.strptime(published_at, "%Y-%m-%dT%H:%M:%SZ").time() published_at, "%Y-%m-%dT%H:%M:%SZ"
).date()
release_time = datetime.strptime(
published_at, "%Y-%m-%dT%H:%M:%SZ"
).time()
writer.writerow([repo_name, tag_name, release_date, release_time]) writer.writerow([repo_name, tag_name, release_date, release_time])
print(f"Release data has been written to {csv_file}") print(f"Release data has been written to {csv_file}")
# Example usage # Example usage
if __name__ == "__main__": if __name__ == "__main__":
fetch_and_write_releases('argo_releases.csv') fetch_and_write_releases("argo_releases.csv")

21
scripts/release-analysis/main.py
View file

@ -1,30 +1,31 @@
from fetch_releases import fetch_and_write_releases
from fetch_helmet_releases import fetch_and_write_helmet_releases
from merge_csvs import merge_csv_files
from plot_graph import plot_time_difference
import os import os
from fetch_helmet_releases import fetch_and_write_helmet_releases
from fetch_releases import fetch_and_write_releases
from merge_csvs import merge_csv_files
from plot_graph import plot_time_difference
# Check there is a github token # Check there is a github token
github_token = os.getenv('GITHUB_TOKEN') github_token = os.getenv("GITHUB_TOKEN")
if not github_token: if not github_token:
raise ValueError("GITHUB_TOKEN environment variable is not set") raise ValueError("GITHUB_TOKEN environment variable is not set")
# Do the thing # Do the thing
print("Fetching releases...") print("Fetching releases...")
fetch_and_write_releases('argo_releases.csv') fetch_and_write_releases("argo_releases.csv")
print("Done") print("Done")
print("Fetching Team Helmet releases...") print("Fetching Team Helmet releases...")
fetch_and_write_helmet_releases('argo_helm_releases.csv') fetch_and_write_helmet_releases("argo_helm_releases.csv")
print("Done") print("Done")
print("Merging release info...") print("Merging release info...")
merge_csv_files('argo_releases.csv', 'argo_helm_releases.csv', 'merged_releases.csv') merge_csv_files("argo_releases.csv", "argo_helm_releases.csv", "merged_releases.csv")
print("Done") print("Done")
print("Plotting time difference graphs...") print("Plotting time difference graphs...")
plot_time_difference('merged_releases.csv') plot_time_difference("merged_releases.csv")
print("Done") print("Done")
# Delete __pycache__ directories # Delete __pycache__ directories
os.system('rm -rf __pycache__') os.system("rm -rf __pycache__")

103
scripts/release-analysis/merge_csvs.py
View file

@ -1,70 +1,98 @@
import csv import csv
from datetime import datetime from datetime import datetime
def merge_csv_files(csv_file_1, csv_file_2, output_csv_file): def merge_csv_files(csv_file_1, csv_file_2, output_csv_file):
# Read the first CSV file into a dictionary # Read the first CSV file into a dictionary
releases_1 = {} releases_1 = {}
with open(csv_file_1, mode='r') as file: with open(csv_file_1, mode="r") as file:
reader = csv.DictReader(file) reader = csv.DictReader(file)
for row in reader: for row in reader:
key = (row['Repository'], row['Release Tag']) key = (row["Repository"], row["Release Tag"])
releases_1[key] = row releases_1[key] = row
# Read the second CSV file and find the oldest release for each appVersion # Read the second CSV file and find the oldest release for each appVersion
oldest_releases = {} oldest_releases = {}
valid_repos = {'argo-cd', 'argo-events', 'argo-workflows', 'argo-rollouts'} valid_repos = {"argo-cd", "argo-events", "argo-workflows", "argo-rollouts"}
with open(csv_file_2, mode='r') as file: with open(csv_file_2, mode="r") as file:
reader = csv.DictReader(file) reader = csv.DictReader(file)
for row in reader: for row in reader:
release_name = row['Release Name'] release_name = row["Release Name"]
repo_name = '-'.join(release_name.split('-')[:-1]) repo_name = "-".join(release_name.split("-")[:-1])
if repo_name in valid_repos: if repo_name in valid_repos:
app_version = row['App Version'] app_version = row["App Version"]
release_datetime = datetime.strptime(f"{row['Release Date']} {row['Release Time']}", "%Y-%m-%d %H:%M:%S") release_datetime = datetime.strptime(
if (repo_name, app_version) not in oldest_releases or release_datetime < oldest_releases[(repo_name, app_version)]['datetime']: f"{row['Release Date']} {row['Release Time']}", "%Y-%m-%d %H:%M:%S"
)
if (
repo_name,
app_version,
) not in oldest_releases or release_datetime < oldest_releases[
(repo_name, app_version)
][
"datetime"
]:
oldest_releases[(repo_name, app_version)] = { oldest_releases[(repo_name, app_version)] = {
'row': row, "row": row,
'datetime': release_datetime "datetime": release_datetime,
} }
# Merge the oldest releases with the first CSV file # Merge the oldest releases with the first CSV file
merged_releases = [] merged_releases = []
for (repo_name, app_version), data in oldest_releases.items(): for (repo_name, app_version), data in oldest_releases.items():
row = data['row'] row = data["row"]
for key, release in releases_1.items(): for key, release in releases_1.items():
if repo_name == release['Repository'] and app_version == release['Release Tag']: if (
time_difference = data['datetime'] - datetime.strptime(f"{release['Release Date']} {release['Release Time']}", "%Y-%m-%d %H:%M:%S") repo_name == release["Repository"]
time_difference_hours = time_difference.total_seconds() / 3600 # Convert to hours and app_version == release["Release Tag"]
):
time_difference = data["datetime"] - datetime.strptime(
f"{release['Release Date']} {release['Release Time']}",
"%Y-%m-%d %H:%M:%S",
)
time_difference_hours = (
time_difference.total_seconds() / 3600
) # Convert to hours
merged_row = { merged_row = {
'Repository': release['Repository'], "Repository": release["Repository"],
'Release Tag': release['Release Tag'], "Release Tag": release["Release Tag"],
'Release Date': release['Release Date'], "Release Date": release["Release Date"],
'Release Time': release['Release Time'], "Release Time": release["Release Time"],
'App Version': app_version, "App Version": app_version,
'Release Name': row['Release Name'], "Release Name": row["Release Name"],
'Release Date 2': row['Release Date'], "Release Date 2": row["Release Date"],
'Release Time 2': row['Release Time'], "Release Time 2": row["Release Time"],
'Time Difference': time_difference_hours "Time Difference": time_difference_hours,
} }
merged_releases.append(merged_row) merged_releases.append(merged_row)
break break
else: else:
merged_row = { merged_row = {
'Repository': repo_name, "Repository": repo_name,
'Release Tag': '', "Release Tag": "",
'Release Date': '', "Release Date": "",
'Release Time': '', "Release Time": "",
'App Version': app_version, "App Version": app_version,
'Release Name': row['Release Name'], "Release Name": row["Release Name"],
'Release Date 2': row['Release Date'], "Release Date 2": row["Release Date"],
'Release Time 2': row['Release Time'], "Release Time 2": row["Release Time"],
'Time Difference': '' "Time Difference": "",
} }
merged_releases.append(merged_row) merged_releases.append(merged_row)
# Write the merged data to a new CSV file # Write the merged data to a new CSV file
with open(output_csv_file, mode='w', newline='') as file: with open(output_csv_file, mode="w", newline="") as file:
fieldnames = ['Repository', 'Release Tag', 'Release Date', 'Release Time', 'App Version', 'Release Name', 'Release Date 2', 'Release Time 2', 'Time Difference'] fieldnames = [
"Repository",
"Release Tag",
"Release Date",
"Release Time",
"App Version",
"Release Name",
"Release Date 2",
"Release Time 2",
"Time Difference",
]
writer = csv.DictWriter(file, fieldnames=fieldnames) writer = csv.DictWriter(file, fieldnames=fieldnames)
writer.writeheader() writer.writeheader()
for row in merged_releases: for row in merged_releases:
@ -72,6 +100,9 @@ def merge_csv_files(csv_file_1, csv_file_2, output_csv_file):
print(f"Merged data has been written to {output_csv_file}") print(f"Merged data has been written to {output_csv_file}")
# Example usage # Example usage
if __name__ == "__main__": if __name__ == "__main__":
merge_csv_files('argo_releases.csv', 'argo_helm_releases.csv', 'merged_releases.csv') merge_csv_files(
"argo_releases.csv", "argo_helm_releases.csv", "merged_releases.csv"
)

41
scripts/release-analysis/plot_graph.py
View file

@ -1,17 +1,24 @@
import csv import csv
import matplotlib.pyplot as plt import matplotlib.pyplot as plt
from packaging import version from packaging import version
def plot_time_difference(csv_file): def plot_time_difference(csv_file):
# Read the CSV file and process the data # Read the CSV file and process the data
data = {'argo-cd': [], 'argo-events': [], 'argo-workflows': [], 'argo-rollouts': []} data = {"argo-cd": [], "argo-events": [], "argo-workflows": [], "argo-rollouts": []}
release_tags = {'argo-cd': [], 'argo-events': [], 'argo-workflows': [], 'argo-rollouts': []} release_tags = {
with open(csv_file, mode='r') as file: "argo-cd": [],
"argo-events": [],
"argo-workflows": [],
"argo-rollouts": [],
}
with open(csv_file, mode="r") as file:
reader = csv.DictReader(file) reader = csv.DictReader(file)
for row in reader: for row in reader:
repo = row['Repository'] repo = row["Repository"]
time_diff_str = row['Time Difference'] time_diff_str = row["Time Difference"]
release_tag = row['Release Tag'] release_tag = row["Release Tag"]
if repo in data and time_diff_str: if repo in data and time_diff_str:
time_diff = float(time_diff_str) time_diff = float(time_diff_str)
data[repo].append(time_diff) data[repo].append(time_diff)
@ -19,27 +26,33 @@ def plot_time_difference(csv_file):
# Sort the release tags based on semantic versioning # Sort the release tags based on semantic versioning
for repo in release_tags: for repo in release_tags:
sorted_indices = sorted(range(len(release_tags[repo])), key=lambda i: version.parse(release_tags[repo][i])) sorted_indices = sorted(
range(len(release_tags[repo])),
key=lambda i: version.parse(release_tags[repo][i]),
)
release_tags[repo] = [release_tags[repo][i] for i in sorted_indices] release_tags[repo] = [release_tags[repo][i] for i in sorted_indices]
data[repo] = [data[repo][i] for i in sorted_indices] data[repo] = [data[repo][i] for i in sorted_indices]
# Plot the data # Plot the data
for repo, time_diffs in data.items(): for repo, time_diffs in data.items():
plt.figure(figsize=(10, 6)) plt.figure(figsize=(10, 6))
plt.plot(release_tags[repo], time_diffs, marker='o', label=repo) plt.plot(release_tags[repo], time_diffs, marker="o", label=repo)
plt.axhline(y=72, color='r', linestyle='--', label='SLA (72 hours)') plt.axhline(y=72, color="r", linestyle="--", label="SLA (72 hours)")
plt.xlabel('Upstream Release Tag') plt.xlabel("Upstream Release Tag")
plt.ylabel('Time difference between upstream release and Helm Chart release (hours)') plt.ylabel(
plt.title(f'Time to Release Helm Chart for {repo}') "Time difference between upstream release and Helm Chart release (hours)"
)
plt.title(f"Time to Release Helm Chart for {repo}")
plt.legend() plt.legend()
plt.grid(True) plt.grid(True)
plt.xticks(rotation=45) plt.xticks(rotation=45)
plt.tight_layout() plt.tight_layout()
plt.savefig(f'time_difference_plot_{repo}.png') plt.savefig(f"time_difference_plot_{repo}.png")
plt.close() plt.close()
print("The plots have been saved as 'time_difference_plot_<repo>.png'") print("The plots have been saved as 'time_difference_plot_<repo>.png'")
# Example usage # Example usage
if __name__ == "__main__": if __name__ == "__main__":
plot_time_difference('merged_releases.csv') plot_time_difference("merged_releases.csv")