Skip to content

Commit 261e0eb

Browse files
fstasifstasi
committed
Merge remote-tracking branch 'origin/main' into theia-upgrade
# Conflicts: # package.json # yarn.lock
2 parents bf00e6b + 7f8b227 commit 261e0eb

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

49 files changed

+4092
-1313
lines changed

‎.github/ISSUE_TEMPLATE/bug_report.md

+1-1
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
name: Bug report
33
about: Create a report to help us improve
44
title: ''
5-
labels: 'type: bug'
5+
labels: 'type: imperfection'
66
assignees: ''
77

88
---

‎.github/label-configuration-files/labels.yml

+24
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
# Used by the "Sync Labels" workflow
2+
# See: https://github.com/Financial-Times/github-label-sync#label-config-file
3+
4+
- name: "topic: accessibility"
5+
color: "00ffff"
6+
description: Enabling the use of the software by everyone
7+
- name: "topic: CLI"
8+
color: "00ffff"
9+
description: Related to Arduino CLI
10+
- name: "topic: debugger"
11+
color: "00ffff"
12+
description: Related to the integrated debugger
13+
- name: "topic: language server"
14+
color: "00ffff"
15+
description: Related to the Arduino Language Server
16+
- name: "topic: serial monitor"
17+
color: "00ffff"
18+
description: Related to the Serial Monitor
19+
- name: "topic: theia"
20+
color: "00ffff"
21+
description: Related to the Theia IDE framework
22+
- name: "topic: theme"
23+
color: "00ffff"
24+
description: Related to GUI theming

‎.github/tools/fetch_athena_stats.py

+131
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,131 @@
1+
import boto3
2+
import semver
3+
import os
4+
import logging
5+
import uuid
6+
import time
7+
8+
9+
# logging.basicConfig(stream=sys.stdout, level=logging.DEBUG)
10+
log = logging.getLogger()
11+
logging.getLogger("boto3").setLevel(logging.CRITICAL)
12+
logging.getLogger("botocore").setLevel(logging.CRITICAL)
13+
logging.getLogger("urllib3").setLevel(logging.CRITICAL)
14+
15+
16+
def execute(client, statement, dest_s3_output_location):
17+
log.info("execute query: {} dumping in {}".format(statement, dest_s3_output_location))
18+
result = client.start_query_execution(
19+
QueryString=statement,
20+
ClientRequestToken=str(uuid.uuid4()),
21+
ResultConfiguration={
22+
"OutputLocation": dest_s3_output_location,
23+
},
24+
)
25+
execution_id = result["QueryExecutionId"]
26+
log.info("wait for query {} completion".format(execution_id))
27+
wait_for_query_execution_completion(client, execution_id)
28+
log.info("operation successful")
29+
return execution_id
30+
31+
32+
def wait_for_query_execution_completion(client, query_execution_id):
33+
query_ended = False
34+
while not query_ended:
35+
query_execution = client.get_query_execution(QueryExecutionId=query_execution_id)
36+
state = query_execution["QueryExecution"]["Status"]["State"]
37+
if state == "SUCCEEDED":
38+
query_ended = True
39+
elif state in ["FAILED", "CANCELLED"]:
40+
raise BaseException(
41+
"query failed or canceled: {}".format(query_execution["QueryExecution"]["Status"]["StateChangeReason"])
42+
)
43+
else:
44+
time.sleep(1)
45+
46+
47+
def valid(key):
48+
split = key.split("_")
49+
if len(split) < 1:
50+
return False
51+
try:
52+
semver.parse(split[0])
53+
except ValueError:
54+
return False
55+
return True
56+
57+
58+
def get_results(client, execution_id):
59+
results_paginator = client.get_paginator("get_query_results")
60+
results_iter = results_paginator.paginate(QueryExecutionId=execution_id, PaginationConfig={"PageSize": 1000})
61+
res = {}
62+
for results_page in results_iter:
63+
for row in results_page["ResultSet"]["Rows"][1:]:
64+
# Loop through the JSON objects
65+
key = row["Data"][0]["VarCharValue"]
66+
if valid(key):
67+
res[key] = row["Data"][1]["VarCharValue"]
68+
69+
return res
70+
71+
72+
def convert_data(data):
73+
result = []
74+
for key, value in data.items():
75+
# 0.18.0_macOS_64bit.tar.gz
76+
split_key = key.split("_")
77+
if len(split_key) != 3:
78+
continue
79+
(version, os_version, arch) = split_key
80+
arch_split = arch.split(".")
81+
if len(arch_split) < 1:
82+
continue
83+
arch = arch_split[0]
84+
if len(arch) > 10:
85+
# This can't be an architecture really.
86+
# It's an ugly solution but works for now so deal with it.
87+
continue
88+
repo = os.environ["GITHUB_REPOSITORY"].split("/")[1]
89+
result.append(
90+
{
91+
"type": "gauge",
92+
"name": "arduino.downloads.total",
93+
"value": value,
94+
"host": os.environ["GITHUB_REPOSITORY"],
95+
"tags": [
96+
f"version:{version}",
97+
f"os:{os_version}",
98+
f"arch:{arch}",
99+
"cdn:downloads.arduino.cc",
100+
f"project:{repo}",
101+
],
102+
}
103+
)
104+
105+
return result
106+
107+
108+
if __name__ == "__main__":
109+
DEST_S3_OUTPUT = os.environ["AWS_ATHENA_OUTPUT_LOCATION"]
110+
AWS_ATHENA_SOURCE_TABLE = os.environ["AWS_ATHENA_SOURCE_TABLE"]
111+
112+
session = boto3.session.Session(region_name="us-east-1")
113+
athena_client = session.client("athena")
114+
115+
# Load all partitions before querying downloads
116+
execute(athena_client, f"MSCK REPAIR TABLE {AWS_ATHENA_SOURCE_TABLE};", DEST_S3_OUTPUT)
117+
118+
query = f"""SELECT replace(json_extract_scalar(url_decode(url_decode(querystring)),
119+
'$.data.url'), 'https://downloads.arduino.cc/arduino-ide/arduino-ide_', '')
120+
AS flavor, count(json_extract(url_decode(url_decode(querystring)),'$')) AS gauge
121+
FROM {AWS_ATHENA_SOURCE_TABLE}
122+
WHERE json_extract_scalar(url_decode(url_decode(querystring)),'$.data.url')
123+
LIKE 'https://downloads.arduino.cc/arduino-ide/arduino-ide_%'
124+
AND json_extract_scalar(url_decode(url_decode(querystring)),'$.data.url')
125+
NOT LIKE '%latest%' -- exclude latest redirect
126+
group by 1 ;"""
127+
exec_id = execute(athena_client, query, DEST_S3_OUTPUT)
128+
results = get_results(athena_client, exec_id)
129+
result_json = convert_data(results)
130+
131+
print(f"::set-output name=result::{result_json}")

‎.github/workflows/arduino-stats.yaml

+57
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
name: arduino-stats
2+
3+
on:
4+
schedule:
5+
# run every day at 07:00 AM, 03:00 PM and 11:00 PM
6+
- cron: "0 7,15,23 * * *"
7+
workflow_dispatch:
8+
repository_dispatch:
9+
10+
jobs:
11+
push-stats:
12+
# This workflow is only of value to the arduino/arduino-ide repository and
13+
# would always fail in forks
14+
if: github.repository == 'arduino/arduino-ide'
15+
runs-on: ubuntu-latest
16+
17+
steps:
18+
- name: Checkout
19+
uses: actions/checkout@v2
20+
21+
- uses: actions/setup-python@v2
22+
with:
23+
python-version: '3.x'
24+
25+
- name: Fetch downloads count form Arduino CDN using AWS Athena
26+
id: fetch
27+
env:
28+
AWS_ACCESS_KEY_ID: ${{ secrets.STATS_AWS_ACCESS_KEY_ID }}
29+
AWS_SECRET_ACCESS_KEY: ${{ secrets.STATS_AWS_SECRET_ACCESS_KEY }}
30+
AWS_ATHENA_SOURCE_TABLE: ${{ secrets.STATS_AWS_ATHENA_SOURCE_TABLE }}
31+
AWS_ATHENA_OUTPUT_LOCATION: ${{ secrets.STATS_AWS_ATHENA_OUTPUT_LOCATION }}
32+
GITHUB_REPOSITORY: ${{ github.repository }}
33+
run: |
34+
pip install boto3 semver
35+
python .github/tools/fetch_athena_stats.py
36+
37+
- name: Send metrics
38+
uses: masci/datadog@v1
39+
with:
40+
api-key: ${{ secrets.DD_API_KEY }}
41+
# Metrics input expects YAML but JSON will work just right.
42+
metrics: ${{steps.fetch.outputs.result}}
43+
44+
- name: Report failure
45+
if: failure()
46+
uses: masci/datadog@v1
47+
with:
48+
api-key: ${{ secrets.DD_API_KEY }}
49+
events: |
50+
- title: "Arduino IDE stats failing"
51+
text: "Stats collection failed"
52+
alert_type: "error"
53+
host: ${{ github.repository }}
54+
tags:
55+
- "project:arduino-ide"
56+
- "cdn:downloads.arduino.cc"
57+
- "workflow:${{ github.workflow }}"

‎.github/workflows/github-stats.yaml

+96
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,96 @@
1+
name: github-stats
2+
3+
on:
4+
schedule:
5+
# run every 30 minutes
6+
- cron: "*/30 * * * *"
7+
workflow_dispatch:
8+
repository_dispatch:
9+
10+
jobs:
11+
push-stats:
12+
# This workflow is only of value to the arduino/arduino-ide repository and
13+
# would always fail in forks
14+
if: github.repository == 'arduino/arduino-ide'
15+
runs-on: ubuntu-latest
16+
17+
steps:
18+
- name: Fetch downloads count
19+
id: fetch
20+
uses: actions/github-script@v4
21+
with:
22+
github-token: ${{github.token}}
23+
script: |
24+
let metrics = []
25+
26+
// Get a list of releases
27+
const opts = github.repos.listReleases.endpoint.merge({
28+
...context.repo
29+
})
30+
const releases = await github.paginate(opts)
31+
32+
// Get download stats for every release
33+
for (const rel of releases) {
34+
// Names for assets are like `arduino-ide_2.0.0-beta.11_Linux_64bit.zip`,
35+
// we'll use this later to split the asset file name more easily
36+
const baseName = `arduino-ide_${rel.name}_`
37+
38+
// Get a list of assets for this release
39+
const opts = github.repos.listReleaseAssets.endpoint.merge({
40+
...context.repo,
41+
release_id: rel.id
42+
})
43+
const assets = await github.paginate(opts)
44+
45+
for (const asset of assets) {
46+
// Ignore files that are not arduino-ide packages
47+
if (!asset.name.startsWith(baseName)) {
48+
continue
49+
}
50+
51+
// Strip the base and remove file extension to get `Linux_32bit`
52+
systemArch = asset.name.replace(baseName, "").split(".")[0].split("_")
53+
54+
// Add a metric object to the list of gathered metrics
55+
metrics.push({
56+
"type": "gauge",
57+
"name": "arduino.downloads.total",
58+
"value": asset.download_count,
59+
"host": "${{ github.repository }}",
60+
"tags": [
61+
`version:${rel.name}`,
62+
`os:${systemArch[0]}`,
63+
`arch:${systemArch[1]}`,
64+
"cdn:github.com",
65+
"project:arduino-ide"
66+
]
67+
})
68+
}
69+
}
70+
71+
// The action will put whatever we return from this function in
72+
// `outputs.result`, JSON encoded. So we just return the array
73+
// of objects and GitHub will do the rest.
74+
return metrics
75+
76+
- name: Send metrics
77+
uses: masci/datadog@v1
78+
with:
79+
api-key: ${{ secrets.DD_API_KEY }}
80+
# Metrics input expects YAML but JSON will work just right.
81+
metrics: ${{steps.fetch.outputs.result}}
82+
83+
- name: Report failure
84+
if: failure()
85+
uses: masci/datadog@v1
86+
with:
87+
api-key: ${{ secrets.DD_API_KEY }}
88+
events: |
89+
- title: "Arduino IDE stats failing"
90+
text: "Stats collection failed"
91+
alert_type: "error"
92+
host: ${{ github.repository }}
93+
tags:
94+
- "project:arduino-ide"
95+
- "cdn:github.com"
96+
- "workflow:${{ github.workflow }}"

0 commit comments

Comments
 (0)