From 7cc4fee48b6d83a83a77404666dda83b971b59dd Mon Sep 17 00:00:00 2001 From: Michel Roegl-Brunner <73236783+michelroegl-brunner@users.noreply.github.com> Date: Tue, 18 Mar 2025 11:20:28 +0100 Subject: [PATCH] Add worflow to crawl APP verisons (#3192) * Add worflow to crawl verisons * Update Test --- .github/workflows/crawl-versions.yaml | 113 ++++ frontend/public/json/versions.json | 562 ++++++++++++++++++ .../__tests__/public/validate-json.test.ts | 9 +- frontend/src/app/api/categories/route.ts | 3 +- frontend/tsconfig.json | 16 +- 5 files changed, 695 insertions(+), 8 deletions(-) create mode 100644 .github/workflows/crawl-versions.yaml create mode 100644 frontend/public/json/versions.json diff --git a/.github/workflows/crawl-versions.yaml b/.github/workflows/crawl-versions.yaml new file mode 100644 index 000000000..52d25264a --- /dev/null +++ b/.github/workflows/crawl-versions.yaml @@ -0,0 +1,113 @@ +name: Crawl Versions from newreleases.io + +on: + workflow_dispatch: + schedule: + # Runs at 12:00 AM and 12:00 PM UTC + - cron: "0 0,12 * * *" + +permissions: + contents: write + pull-requests: write + +jobs: + move-to-main-repo: + runs-on: runner-cluster-htl-set + + steps: + - name: Checkout Repository + uses: actions/checkout@v2 + with: + repository: community-scripts/ProxmoxVED + ref: main + + - name: Generate a token + id: generate-token + uses: actions/create-github-app-token@v1 + with: + app-id: ${{ vars.APP_ID }} + private-key: ${{ secrets.APP_PRIVATE_KEY }} + + - name: Crawl from newreleases.io + env: + token: ${{ secrets.NEWRELEASES_TOKEN }} + run: | + page=1 + projects_file="project_json" + output_file="frontend/public/json/versions.json" + + echo "[]" > $output_file + + while true; do + + echo "Start loop on page: $page" + + projects=$(curl -s -H "X-Key: $token" "https://api.newreleases.io/v1/projects?page=$page") + total_pages=$(echo "$projects" | jq -r '.total_pages') + + if [ -z "$total_pages" ] || [ "$total_pages" -eq 0 ]; then + echo "No pages available. Exiting." + exit 1 + fi + if [ $page == $total_pages ]; then + break + fi + + if [ -z "$projects" ] || ! echo "$projects" | jq -e '.projects' > /dev/null; then + echo "No more projects or invalid response. Exiting." + break + fi + + echo "$projects" > "$projects_file" + + jq -r '.projects[] | "\(.id) \(.name)"' "$projects_file" | while read -r id name; do + version=$(curl -s -H "X-Key: $token" "https://api.newreleases.io/v1/projects/$id/latest-release") + version_data=$(echo "$version" | jq -r '.version // empty') + if [ -n "$version_data" ]; then + jq --arg name "$name" --arg version "$version_data" \ + '. += [{"name": $name, "version": $version}]' "$output_file" > "$output_file.tmp" && mv "$output_file.tmp" "$output_file" + fi + done + ((page++)) + done + + - name: Commit JSON + env: + GH_TOKEN: ${{ steps.generate-token.outputs.token }} + run: | + git config --global user.email "github-actions[bot]@users.noreply.github.com" + git config --global user.name "GitHub Actions[bot]" + git checkout -b update_versions || git checkout update_versions + git add frontend/public/json/versions.json + if git diff --cached --quiet; then + echo "No changes detected." + echo "changed=false" >> "$GITHUB_ENV" + exit 0 + else + echo "Changes detected:" + git diff --stat --cached + echo "changed=true" >> "$GITHUB_ENV" + fi + git commit -m "Update versions.json" + git push origin update_versions --force + gh pr create --title "[AUTOMATIC PR]Update versions.json" --body "Update versions.json, crawled from newreleases.io" --base main --head update_versions + + - name: Approve pull request + if: env.changed == 'true' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + PR_NUMBER=$(gh pr list --head "update_versions" --json number --jq '.[].number') + if [ -n "$PR_NUMBER" ]; then + gh pr review $PR_NUMBER --approve + fi + + - name: Re-approve pull request after update + if: env.changed == 'true' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + PR_NUMBER=$(gh pr list --head "update_versions" --json number --jq '.[].number') + if [ -n "$PR_NUMBER" ]; then + gh pr review $PR_NUMBER --approve + fi diff --git a/frontend/public/json/versions.json b/frontend/public/json/versions.json new file mode 100644 index 000000000..54bdced0b --- /dev/null +++ b/frontend/public/json/versions.json @@ -0,0 +1,562 @@ +[ + { + "name": "Jackett/Jackett", + "version": "v0.22.1660" + }, + { + "name": "authelia/authelia", + "version": "v4.39.1" + }, + { + "name": "ollama/ollama", + "version": "v0.6.1" + }, + { + "name": "advplyr/audiobookshelf", + "version": "v2.20.0" + }, + { + "name": "bastienwirtz/homer", + "version": "v25.03.3" + }, + { + "name": "keycloak/keycloak", + "version": "26.1.4" + }, + { + "name": "semaphoreui/semaphore", + "version": "v2.13.0" + }, + { + "name": "Kareadita/Kavita", + "version": "v0.8.5.11" + }, + { + "name": "TriliumNext/Notes", + "version": "v0.92.4" + }, + { + "name": "morpheus65535/bazarr", + "version": "v1.5.1" + }, + { + "name": "fallenbagel/jellyseerr", + "version": "v2.5.1" + }, + { + "name": "firefly-iii/firefly-iii", + "version": "develop-20250222.4" + }, + { + "name": "glanceapp/glance", + "version": "v0.7.7" + }, + { + "name": "gethomepage/homepage", + "version": "v1.0.4" + }, + { + "name": "Donkie/Spoolman", + "version": "v0.22.0" + }, + { + "name": "pocket-id/pocket-id", + "version": "v0.40.1" + }, + { + "name": "evcc-io/evcc", + "version": "0.201.0" + }, + { + "name": "BookStackApp/BookStack", + "version": "v25.02.1" + }, + { + "name": "hoarder-app/hoarder", + "version": "ios/v1.6.9-0" + }, + { + "name": "pocketbase/pocketbase", + "version": "v0.26.1" + }, + { + "name": "Kozea/Radicale", + "version": "v3.5.0" + }, + { + "name": "VictoriaMetrics/VictoriaMetrics", + "version": "v1.17.0-victorialogs" + }, + { + "name": "msgbyte/tianji", + "version": "v1.18.22" + }, + { + "name": "duplicati/duplicati", + "version": "v2.1.0.111-2.1.0.111_canary_2025-03-15" + }, + { + "name": "henrygd/beszel", + "version": "v0.10.2" + }, + { + "name": "navidrome/navidrome", + "version": "v0.55.1" + }, + { + "name": "Threadfin/Threadfin", + "version": "1.2.31" + }, + { + "name": "Stirling-Tools/Stirling-PDF", + "version": "v0.44.2" + }, + { + "name": "ipfs/kubo", + "version": "v0.33.2" + }, + { + "name": "paperless-ngx/paperless-ngx", + "version": "v2.15.0-beta" + }, + { + "name": "wazuh/wazuh", + "version": "coverity-w11-4.12.0" + }, + { + "name": "homarr-labs/homarr", + "version": "v1.11.0" + }, + { + "name": "docker/compose", + "version": "v2.34.0" + }, + { + "name": "FreshRSS/FreshRSS", + "version": "1.26.1" + }, + { + "name": "kimai/kimai", + "version": "2.31.0" + }, + { + "name": "zitadel/zitadel", + "version": "v2.66.13" + }, + { + "name": "usememos/memos", + "version": "v0.24.1" + }, + { + "name": "openobserve/openobserve", + "version": "v0.14.5-rc3" + }, + { + "name": "NodeBB/NodeBB", + "version": "v4.1.1" + }, + { + "name": "diced/zipline", + "version": "v4.0.1" + }, + { + "name": "minio/minio", + "version": "RELEASE.2025-03-12T18-04-18Z" + }, + { + "name": "zwave-js/zwave-js-ui", + "version": "v9.33.0" + }, + { + "name": "cockpit-project/cockpit", + "version": "335" + }, + { + "name": "gotson/komga", + "version": "1.21.2" + }, + { + "name": "benjaminjonard/koillection", + "version": "1.6.12" + }, + { + "name": "excalidraw/excalidraw", + "version": "v0.18.0" + }, + { + "name": "Ombi-app/Ombi", + "version": "v4.47.1" + }, + { + "name": "mylar3/mylar3", + "version": "v0.8.2" + }, + { + "name": "stonith404/pingvin-share", + "version": "v1.10.3" + }, + { + "name": "AdguardTeam/AdGuardHome", + "version": "v0.107.57" + }, + { + "name": "Luligu/matterbridge", + "version": "2.2.4" + }, + { + "name": "AlexxIT/go2rtc", + "version": "v1.9.9" + }, + { + "name": "clusterzx/paperless-ai", + "version": "v2.7.4" + }, + { + "name": "YuukanOO/seelf", + "version": "v2.4.2" + }, + { + "name": "umami-software/umami", + "version": "v2.17.0" + }, + { + "name": "documenso/documenso", + "version": "v1.9.1-rc.9" + }, + { + "name": "rogerfar/rdt-client", + "version": "v2.0.102" + }, + { + "name": "prometheus/alertmanager", + "version": "v0.28.1" + }, + { + "name": "hargata/lubelog", + "version": "v1.4.5" + }, + { + "name": "ellite/Wallos", + "version": "v2.46.1" + }, + { + "name": "Dolibarr/dolibarr", + "version": "21.0.0" + }, + { + "name": "netbox-community/netbox", + "version": "v4.2.5" + }, + { + "name": "open-webui/open-webui", + "version": "v0.5.20" + }, + { + "name": "matze/wastebin", + "version": "3.0.0" + }, + { + "name": "immich-app/immich", + "version": "v1.129.0" + }, + { + "name": "snipe/snipe-it", + "version": "v8.0.4" + }, + { + "name": "toniebox-reverse-engineering/teddycloud", + "version": "tc_v0.6.4" + }, + { + "name": "go-gitea/gitea", + "version": "v1.23.5" + }, + { + "name": "sysadminsmedia/homebox", + "version": "v0.18.0" + }, + { + "name": "Koenkk/zigbee2mqtt", + "version": "2.1.3" + }, + { + "name": "heiher/hev-socks5-server", + "version": "2.8.0" + }, + { + "name": "inspircd/inspircd", + "version": "v4.6.0" + }, + { + "name": "tobychui/zoraxy", + "version": "v3.1.9" + }, + { + "name": "grocy/grocy", + "version": "v4.4.2" + }, + { + "name": "jhuckaby/Cronicle", + "version": "v0.9.76" + }, + { + "name": "docmost/docmost", + "version": "v0.8.4" + }, + { + "name": "Part-DB/Part-DB-server", + "version": "v1.16.1" + }, + { + "name": "prometheus/prometheus", + "version": "v3.2.1" + }, + { + "name": "silverbulletmd/silverbullet", + "version": "0.10.4" + }, + { + "name": "juanfont/headscale", + "version": "v0.25.1" + }, + { + "name": "benzino77/tasmocompiler", + "version": "v12.5.0" + }, + { + "name": "traefik/traefik", + "version": "v3.3.4" + }, + { + "name": "schlagmichdoch/PairDrop", + "version": "v1.11.2" + }, + { + "name": "Athou/commafeed", + "version": "5.6.1" + }, + { + "name": "azukaar/Cosmos-Server", + "version": "v0.18.3" + }, + { + "name": "wavelog/wavelog", + "version": "2.0.1" + }, + { + "name": "sabnzbd/sabnzbd", + "version": "4.4.1" + }, + { + "name": "grafana/grafana", + "version": "v11.5.2" + }, + { + "name": "gristlabs/grist-core", + "version": "v1.4.2" + }, + { + "name": "prometheus-pve/prometheus-pve-exporter", + "version": "v3.5.2" + }, + { + "name": "sbondCo/Watcharr", + "version": "v2.0.2" + }, + { + "name": "glpi-project/glpi", + "version": "10.0.18" + }, + { + "name": "TasmoAdmin/TasmoAdmin", + "version": "v4.2.3" + }, + { + "name": "dani-garcia/vaultwarden", + "version": "1.33.2" + }, + { + "name": "blakeblackshear/frigate", + "version": "v0.15.0" + }, + { + "name": "bluenviron/mediamtx", + "version": "v1.11.3" + }, + { + "name": "actualbudget/actual-server", + "version": "v25.2.1" + }, + { + "name": "NginxProxyManager/nginx-proxy-manager", + "version": "v2.12.3" + }, + { + "name": "thomiceli/opengist", + "version": "v1.9.1" + }, + { + "name": "Forceu/Gokapi", + "version": "v1.9.6" + }, + { + "name": "PrivateBin/PrivateBin", + "version": "1.7.6" + }, + { + "name": "hivemq/hivemq-community-edition", + "version": "2025.1" + }, + { + "name": "rustdesk/rustdesk-server", + "version": "1.1.14" + }, + { + "name": "hansmi/prometheus-paperless-exporter", + "version": "v0.0.6" + }, + { + "name": "donaldzou/WGDashboard", + "version": "v4.1.4" + }, + { + "name": "0xERR0R/blocky", + "version": "v0.25" + }, + { + "name": "linkwarden/linkwarden", + "version": "v2.9.3" + }, + { + "name": "Tautulli/Tautulli", + "version": "v2.15.1" + }, + { + "name": "traccar/traccar", + "version": "v6.6" + }, + { + "name": "ErsatzTV/ErsatzTV", + "version": "v25.1.0" + }, + { + "name": "seanmorley15/AdventureLog", + "version": "v0.8.0" + }, + { + "name": "MagicMirrorOrg/MagicMirror", + "version": "v2.30.0" + }, + { + "name": "louislam/uptime-kuma", + "version": "2.0.0-beta.1" + }, + { + "name": "pymedusa/Medusa", + "version": "v1.0.22" + }, + { + "name": "phpipam/phpipam", + "version": "v1.7.3" + }, + { + "name": "Bubka/2FAuth", + "version": "v5.4.3" + }, + { + "name": "gotify/server", + "version": "v2.6.1" + }, + { + "name": "janeczku/calibre-web", + "version": "0.6.24" + }, + { + "name": "sabre-io/Baikal", + "version": "0.10.1" + }, + { + "name": "caddyserver/xcaddy", + "version": "v0.4.4" + }, + { + "name": "linuxserver/Heimdall", + "version": "v2.6.3" + }, + { + "name": "aceberg/WatchYourLAN", + "version": "2.0.4" + }, + { + "name": "Kometa-Team/Kometa", + "version": "v2.1.0" + }, + { + "name": "FunkeyFlo/ps5-mqtt", + "version": "v1.4.0" + }, + { + "name": "projectsend/projectsend", + "version": "r1720" + }, + { + "name": "Pf2eToolsOrg/Pf2eTools", + "version": "v0.8.13" + }, + { + "name": "Paymenter/Paymenter", + "version": "v0.9.5" + }, + { + "name": "hywax/mafl", + "version": "v0.15.4" + }, + { + "name": "FlareSolverr/FlareSolverr", + "version": "v3.3.21" + }, + { + "name": "Suwayomi/Suwayomi-Server", + "version": "v1.1.1" + }, + { + "name": "Forceu/barcodebuddy", + "version": "v1.8.1.8" + }, + { + "name": "Lissy93/dashy", + "version": "3.1.1" + }, + { + "name": "gnmyt/myspeed", + "version": "v1.0.9" + }, + { + "name": "monicahq/monica", + "version": "v4.1.2" + }, + { + "name": "thelounge/thelounge-deb", + "version": "v4.4.3" + }, + { + "name": "wger-project/wger", + "version": "2.2" + }, + { + "name": "sct/overseerr", + "version": "preview-test-node-18" + }, + { + "name": "deepch/RTSPtoWeb", + "version": "v2.4.3" + }, + { + "name": "searxng/searxng", + "version": "v1.0.0" + }, + { + "name": "MediaBrowser/Emby", + "version": "3.5.2.0" + } +] diff --git a/frontend/src/__tests__/public/validate-json.test.ts b/frontend/src/__tests__/public/validate-json.test.ts index 562ebe9c1..1ab52db68 100644 --- a/frontend/src/__tests__/public/validate-json.test.ts +++ b/frontend/src/__tests__/public/validate-json.test.ts @@ -3,13 +3,14 @@ import { promises as fs } from "fs"; import path from "path"; import { ScriptSchema, type Script } from "@/app/json-editor/_schemas/schemas"; import { Metadata } from "@/lib/types"; - +console.log('Current directory: ' + process.cwd()); const jsonDir = "public/json"; const metadataFileName = "metadata.json"; +const versionsFileName = "versions.json"; const encoding = "utf-8"; const fileNames = (await fs.readdir(jsonDir)) - .filter((fileName) => fileName !== metadataFileName) + .filter((fileName) => fileName !== metadataFileName && fileName !== versionsFileName); describe.each(fileNames)("%s", async (fileName) => { let script: Script; @@ -20,6 +21,7 @@ describe.each(fileNames)("%s", async (fileName) => { script = JSON.parse(fileContent); }) + it("should have valid json according to script schema", () => { ScriptSchema.parse(script); }); @@ -27,6 +29,8 @@ describe.each(fileNames)("%s", async (fileName) => { it("should have a corresponding script file", () => { script.install_methods.forEach((method) => { const scriptPath = path.resolve("..", method.script) + //FIXME: Dose note account for new dir structure and files in /script/tools + assert(fs.stat(scriptPath), `Script file not found: ${scriptPath}`) }) }); @@ -40,7 +44,6 @@ describe(`${metadataFileName}`, async () => { const fileContent = await fs.readFile(filePath, encoding) metadata = JSON.parse(fileContent); }) - it("should have valid json according to metadata schema", () => { // TODO: create zod schema for metadata. Move zod schemas to /lib/types.ts assert(metadata.categories.length > 0); diff --git a/frontend/src/app/api/categories/route.ts b/frontend/src/app/api/categories/route.ts index 2d33a3ab0..4db931509 100644 --- a/frontend/src/app/api/categories/route.ts +++ b/frontend/src/app/api/categories/route.ts @@ -7,6 +7,7 @@ export const dynamic = "force-static"; const jsonDir = "public/json"; const metadataFileName = "metadata.json"; +const versionFileName = "version.json"; const encoding = "utf-8"; const getMetadata = async () => { @@ -18,7 +19,7 @@ const getMetadata = async () => { const getScripts = async () => { const filePaths = (await fs.readdir(jsonDir)) - .filter((fileName) => fileName !== metadataFileName) + .filter((fileName) => fileName !== metadataFileName && fileName !== versionFileName) .map((fileName) => path.resolve(jsonDir, fileName)); const scripts = await Promise.all( diff --git a/frontend/tsconfig.json b/frontend/tsconfig.json index 4b269df34..195f88b78 100644 --- a/frontend/tsconfig.json +++ b/frontend/tsconfig.json @@ -1,6 +1,10 @@ { "compilerOptions": { - "lib": ["dom", "dom.iterable", "esnext"], + "lib": [ + "dom", + "dom.iterable", + "esnext" + ], "allowJs": true, "skipLibCheck": true, "strict": true, @@ -10,7 +14,7 @@ "moduleResolution": "bundler", "resolveJsonModule": true, "isolatedModules": true, - "jsx": "react-jsx", + "jsx": "preserve", "incremental": true, "plugins": [ { @@ -18,7 +22,9 @@ } ], "paths": { - "@/*": ["./src/*"] + "@/*": [ + "./src/*" + ] }, "target": "ES2017" }, @@ -29,5 +35,7 @@ ".next/types/**/*.ts", "next.config.mjs" ], - "exclude": ["node_modules"] + "exclude": [ + "node_modules" + ] }