diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..2fa1915 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,16 @@ +root = true + +[*] +charset = utf-8 +end_of_line = lf +insert_final_newline = true +trim_trailing_whitespace = true +indent_style = space +indent_size = 4 + +[*.{yml,yaml,json,md}] +indent_size = 2 + +[*.sh] +indent_style = space +indent_size = 4 diff --git a/.github/workflows/bicep_deployment.yaml b/.github/workflows/bicep_deployment.yaml index 9965738..99f697b 100644 --- a/.github/workflows/bicep_deployment.yaml +++ b/.github/workflows/bicep_deployment.yaml @@ -1,9 +1,8 @@ name: Azure Bicep Deployment +# Manual only: the deploy step references ./main.bicep, which does not exist in +# this repo yet. Add a main.bicep and re-enable a push trigger when ready. on: - push: - branches: - - main workflow_dispatch: inputs: resource_group: @@ -41,5 +40,5 @@ jobs: - name: Deploy Infrastructure using Bicep run: | - az group create -l $LOCATION -n $RESOURCE_GROUP - az deployment group create -f ./main.bicep -g $RESOURCE_GROUP + az group create -l "$LOCATION" -n "$RESOURCE_GROUP" + az deployment group create -f ./main.bicep -g "$RESOURCE_GROUP" diff --git a/.github/workflows/linter.yml b/.github/workflows/linter.yml index fa485d5..3b2a50b 100644 --- a/.github/workflows/linter.yml +++ b/.github/workflows/linter.yml @@ -1,6 +1,13 @@ -name: SuperLinter +name: Linter -on: [pull_request] +on: + push: + branches: [main] + pull_request: + +permissions: + contents: read + statuses: write jobs: build: @@ -8,12 +15,26 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout Code - uses: actions/checkout@v2 + uses: actions/checkout@v4 + with: + fetch-depth: 0 - name: Lint Code Base - uses: github/super-linter@v4.2.2 + uses: super-linter/super-linter/slim@v7 env: + # Super-Linter v7 enables ~50 validators by default. This is a curated + # opt-in: only the listed validators run (high-signal for a + # PowerShell / Bash / docs repo). PowerShell is also covered by the + # dedicated scriptanalyzer.yml workflow. VALIDATE_ALL_CODEBASE: false - VALIDATE_MARKDOWN: false DEFAULT_BRANCH: main + LINTER_RULES_PATH: . + VALIDATE_BASH: true + VALIDATE_YAML: true + VALIDATE_GITHUB_ACTIONS: true + VALIDATE_GITLEAKS: true + VALIDATE_MARKDOWN: true + # Exclude the vendored AVM agent playbook and informal cheatsheets from + # linting, plus any local Python venv / bytecode. + FILTER_REGEX_EXCLUDE: '(\.github/agents/|cheatsheets/|\.venv/|__pycache__/|/path/to/)' GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file diff --git a/.github/workflows/scriptanalyzer.yml b/.github/workflows/scriptanalyzer.yml index 591658d..7189849 100644 --- a/.github/workflows/scriptanalyzer.yml +++ b/.github/workflows/scriptanalyzer.yml @@ -1,17 +1,32 @@ name: ScriptAnalyzer -on: [pull_request] +on: + push: + branches: [main] + pull_request: + +permissions: + contents: read jobs: - job: - name: "Run PSSA" + analyze: + name: Run PSScriptAnalyzer runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v2 - - name: ScriptAnalyzer + uses: actions/checkout@v4 + + - name: Run PSScriptAnalyzer shell: pwsh run: | - # $VerbosePreference = "Continue" - Install-Module PSScriptAnalyzer -RequiredVersion 1.19.0 -Force - Invoke-ScriptAnalyzer -Path . -Recurse -ReportSummary + Set-PSRepository PSGallery -InstallationPolicy Trusted + Install-Module PSScriptAnalyzer -Scope CurrentUser -Force + $settings = Join-Path $PWD 'PSScriptAnalyzerSettings.psd1' + $params = @{ Path = '.'; Recurse = $true; ReportSummary = $true } + if (Test-Path $settings) { $params.Settings = $settings } + $results = Invoke-ScriptAnalyzer @params + $results | Format-Table -AutoSize + if ($results | Where-Object Severity -in 'Error', 'Warning') { + Write-Error "PSScriptAnalyzer found $($results.Count) issue(s)." + exit 1 + } diff --git a/.github/workflows/template-sync.yml b/.github/workflows/template-sync.yml new file mode 100644 index 0000000..df3e42f --- /dev/null +++ b/.github/workflows/template-sync.yml @@ -0,0 +1,34 @@ +name: Template Sync + +# Keeps repositories generated from this template in sync with its scaffolding. +# Runs in the generated repo (skipped in the template itself) and opens a PR +# whenever this template changes. Protect repo-specific files via .templatesyncignore. + +on: + schedule: + - cron: '0 6 * * 1' # Mondays 06:00 UTC + workflow_dispatch: + +permissions: + contents: write + pull-requests: write + +jobs: + sync: + name: Sync from template + runs-on: ubuntu-latest + # No-op in the template repository itself. + if: github.repository != 'segraef/Template' + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Sync from template + uses: AndreasAugustin/actions-template-sync@v2 + with: + source_repo_path: segraef/Template + upstream_branch: main + pr_labels: template-sync + pr_title: "chore: sync from segraef/Template" diff --git a/.gitignore b/.gitignore index 7abd19d..9bd9f45 100644 --- a/.gitignore +++ b/.gitignore @@ -10,5 +10,20 @@ !.vscode/extensions.json *.code-workspace -#Python +# Python +__pycache__/ +*.py[cod] +.venv/ +venv/ +env/ .env +*.egg-info/ +.pytest_cache/ + +# Node +node_modules/ +npm-debug.log* + +# Logs / local +*.log +*.tmp diff --git a/.markdownlint.yaml b/.markdownlint.yaml new file mode 100644 index 0000000..2f29452 --- /dev/null +++ b/.markdownlint.yaml @@ -0,0 +1,14 @@ +# markdownlint configuration (consumed by Super-Linter's MARKDOWN validator). +default: true + +# Long lines are fine: tables, badge URLs and reference links exceed 80 cols. +MD013: false + +# Allow repeated sibling headings across sections (e.g. Added/Changed/Fixed per +# CHANGELOG release). +MD024: + siblings_only: true + +# Badges and icons use inline HTML / images in headings. +MD033: false +MD041: false diff --git a/.pipelines/variables.yml b/.pipelines/variables.yml index 71c15c5..02d2362 100644 --- a/.pipelines/variables.yml +++ b/.pipelines/variables.yml @@ -1,8 +1,8 @@ variables: vmImage: 'ubuntu-latest' poolName: '' - serviceConnection: '' location: 'West Europe' - resourceGroupName: '' azurePowerShellVersion: 'latestVersion' preferredAzurePowerShellVersion: '' diff --git a/.templatesyncignore b/.templatesyncignore new file mode 100644 index 0000000..5547bff --- /dev/null +++ b/.templatesyncignore @@ -0,0 +1,21 @@ +# Files Template Sync must never overwrite in this repository. +# Syntax follows .gitignore semantics (AndreasAugustin/actions-template-sync). + +# Repository identity and history. +README.md +CHANGELOG.md +LICENSE +.github/CODEOWNERS +.github/workflows/template-sync.yml + +# This repository's own code and content. The template only supplies the +# shared scaffolding (_Template.ps1, Write-Log.psm1, _Template.sh, log.sh, +# PSScriptAnalyzerSettings.psd1, linter/scriptanalyzer workflows), which is +# intentionally NOT ignored so it keeps tracking the template. +PowerShell/Snippets/** +Python/** +flask/** +JavaScript/** +REST/** +graphql/** +cheatsheets/** diff --git a/Bash/_ b/Bash/_ deleted file mode 100644 index 4287ca8..0000000 --- a/Bash/_ +++ /dev/null @@ -1 +0,0 @@ -# \ No newline at end of file diff --git a/Bash/_Template.sh b/Bash/_Template.sh new file mode 100644 index 0000000..c6d7f37 --- /dev/null +++ b/Bash/_Template.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +# +# verb-noun.sh - one-line summary of what this script does. +# +# Description: what it automates, prerequisites (CLIs, auth) and side effects. +# Usage: ./verb-noun.sh +# Author: Sebastian Gräf +# Repo: https://github.com/segraef/Scripts + +set -euo pipefail + +script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" +# shellcheck source-path=SCRIPTDIR source=log.sh +. "${script_dir}/log.sh" + +main() { + local name="${1:-world}" + + log_info "Executing $(basename -- "${BASH_SOURCE[0]}")." + log_info "Hello, ${name}." + log_info "Finished." +} + +main "$@" diff --git a/Bash/avm_manual_analysis.sh b/Bash/avm_manual_analysis.sh index d265f8e..69a6dd2 100755 --- a/Bash/avm_manual_analysis.sh +++ b/Bash/avm_manual_analysis.sh @@ -1,4 +1,23 @@ -#!/bin/bash +#!/usr/bin/env bash +# +# avm_manual_analysis.sh - analyse how AVM pattern module tests/examples use resources. +# +# Description: Scans the Bicep (avm/ptn) pattern module test files and the Terraform +# (terraform-azurerm-avm-ptn*) example files under /Azure, counting how many +# use native Microsoft/azurerm resources versus AVM modules, then prints a +# statistical summary, a Markdown comparison table, and a conclusion. +# Requires the repositories cloned under /Azure and the bc calculator. +# Usage: ./avm_manual_analysis.sh +# Author: Sebastian Gräf +# Repo: https://github.com/segraef/Scripts + +set -euo pipefail + +script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" +# shellcheck source-path=SCRIPTDIR source=log.sh +. "${script_dir}/log.sh" + +log_info "Executing $(basename -- "${BASH_SOURCE[0]}")." echo "=========================================================" echo "AVM Pattern Module Tests & Examples Usage Analysis" @@ -9,42 +28,42 @@ echo echo "1. BICEP PATTERN MODULE TESTS ANALYSIS" echo "======================================" -# Count actual Bicep pattern modules -bicep_modules=$(ls -d /Azure/bicep-registry-modules/avm/ptn/*/* 2>/dev/null | wc -l | tr -d ' ') -echo "Total Bicep pattern modules: $bicep_modules" +# Count actual Bicep pattern modules (avm/ptn// directories) +bicep_modules=$(find /Azure/bicep-registry-modules/avm/ptn -mindepth 2 -maxdepth 2 -type d 2>/dev/null | wc -l | tr -d ' ') +echo "Total Bicep pattern modules: ${bicep_modules}" # Count Bicep files bicep_total=$(find /Azure/bicep-registry-modules/avm/ptn -path "*/tests/*" -name "*.bicep" 2>/dev/null | wc -l | tr -d ' ') -echo "Total Bicep TEST files analyzed: $bicep_total" +echo "Total Bicep TEST files analyzed: ${bicep_total}" # Count files with native resources bicep_native_files=$(find /Azure/bicep-registry-modules/avm/ptn -path "*/tests/*" -name "*.bicep" -exec grep -l "resource.*'Microsoft\." {} \; 2>/dev/null | wc -l | tr -d ' ') -echo "Test files with native Microsoft resources: $bicep_native_files" +echo "Test files with native Microsoft resources: ${bicep_native_files}" # Count files with AVM modules bicep_avm_files=$(find /Azure/bicep-registry-modules/avm/ptn -path "*/tests/*" -name "*.bicep" -exec grep -l "module.*'br/public:avm\|module.*'br:mcr\.microsoft\.com.*avm" {} \; 2>/dev/null | wc -l | tr -d ' ') -echo "Test files with AVM modules: $bicep_avm_files" +echo "Test files with AVM modules: ${bicep_avm_files}" echo echo "2. TERRAFORM PATTERN MODULE EXAMPLES ANALYSIS" echo "==============================================" -# Count actual Terraform pattern modules -terraform_modules=$(ls -d /Azure/terraform-azurerm-avm-ptn*/ 2>/dev/null | wc -l | tr -d ' ') -echo "Total Terraform pattern modules: $terraform_modules" +# Count actual Terraform pattern modules (terraform-azurerm-avm-ptn* directories) +terraform_modules=$(find /Azure -mindepth 1 -maxdepth 1 -type d -name "terraform-azurerm-avm-ptn*" 2>/dev/null | wc -l | tr -d ' ') +echo "Total Terraform pattern modules: ${terraform_modules}" # Count Terraform files terraform_total=$(find /Azure/terraform-azurerm-avm-ptn* -path "*/examples/*" -name "main.tf" 2>/dev/null | wc -l | tr -d ' ') -echo "Total Terraform EXAMPLE files analyzed: $terraform_total" +echo "Total Terraform EXAMPLE files analyzed: ${terraform_total}" # Count files with native resources terraform_native_files=$(find /Azure/terraform-azurerm-avm-ptn* -path "*/examples/*" -name "main.tf" -exec grep -l 'resource "azurerm_' {} \; 2>/dev/null | wc -l | tr -d ' ') -echo "Example files with native azurerm resources: $terraform_native_files" +echo "Example files with native azurerm resources: ${terraform_native_files}" # Count files with AVM modules terraform_avm_files=$(find /Azure/terraform-azurerm-avm-ptn* -path "*/examples/*" -name "main.tf" -exec grep -l 'source.*=.*"Azure/avm-' {} \; 2>/dev/null | wc -l | tr -d ' ') -echo "Example files with AVM modules: $terraform_avm_files" +echo "Example files with AVM modules: ${terraform_avm_files}" echo @@ -52,46 +71,60 @@ echo "3. STATISTICAL SUMMARY" echo "======================" # Calculate percentages for Bicep (file-based analysis) -if [ $bicep_total -gt 0 ]; then - bicep_native_pct=$(echo "scale=1; $bicep_native_files * 100 / $bicep_total" | bc -l) - bicep_avm_pct=$(echo "scale=1; $bicep_avm_files * 100 / $bicep_total" | bc -l) +if [ "${bicep_total}" -gt 0 ]; then + bicep_native_pct=$(echo "scale=1; ${bicep_native_files} * 100 / ${bicep_total}" | bc -l) + bicep_avm_pct=$(echo "scale=1; ${bicep_avm_files} * 100 / ${bicep_total}" | bc -l) else bicep_native_pct=0 bicep_avm_pct=0 fi # Calculate percentages for Terraform (file-based analysis) -if [ $terraform_total -gt 0 ]; then - terraform_native_pct=$(echo "scale=1; $terraform_native_files * 100 / $terraform_total" | bc -l) - terraform_avm_pct=$(echo "scale=1; $terraform_avm_files * 100 / $terraform_total" | bc -l) +if [ "${terraform_total}" -gt 0 ]; then + terraform_native_pct=$(echo "scale=1; ${terraform_native_files} * 100 / ${terraform_total}" | bc -l) + terraform_avm_pct=$(echo "scale=1; ${terraform_avm_files} * 100 / ${terraform_total}" | bc -l) else terraform_native_pct=0 terraform_avm_pct=0 fi +# Pre-compute combined totals for the table and downstream stats. +combined_modules=$((bicep_modules + terraform_modules)) +combined_total=$((bicep_total + terraform_total)) +combined_native_files=$((bicep_native_files + terraform_native_files)) +combined_avm_files=$((bicep_avm_files + terraform_avm_files)) + +if [ "${combined_total}" -gt 0 ]; then + combined_native_pct=$(echo "scale=1; ${combined_native_files} * 100 / ${combined_total}" | bc -l) + combined_avm_pct=$(echo "scale=1; ${combined_avm_files} * 100 / ${combined_total}" | bc -l) +else + combined_native_pct=0 + combined_avm_pct=0 +fi + echo "MARKDOWN TABLE" echo "==============" echo echo "| Ecosystem | Pattern Modules | Test/Example Files | Native Resources | AVM Modules | Native % | AVM % |" echo "|-----------|----------------|-------------------|------------------|-------------|----------|-------|" -echo "| Bicep (Tests) | $bicep_modules | $bicep_total | $bicep_native_files | $bicep_avm_files | ${bicep_native_pct}% | ${bicep_avm_pct}% |" -echo "| Terraform (Examples) | $terraform_modules | $terraform_total | $terraform_native_files | $terraform_avm_files | ${terraform_native_pct}% | ${terraform_avm_pct}% |" -echo "| **Total** | **$((bicep_modules + terraform_modules))** | **$((bicep_total + terraform_total))** | **$((bicep_native_files + terraform_native_files))** | **$((bicep_avm_files + terraform_avm_files))** | **$(echo "scale=1; ($bicep_native_files + $terraform_native_files) * 100 / ($bicep_total + $terraform_total)" | bc -l)%** | **$(echo "scale=1; ($bicep_avm_files + $terraform_avm_files) * 100 / ($bicep_total + $terraform_total)" | bc -l)%** |" +echo "| Bicep (Tests) | ${bicep_modules} | ${bicep_total} | ${bicep_native_files} | ${bicep_avm_files} | ${bicep_native_pct}% | ${bicep_avm_pct}% |" +echo "| Terraform (Examples) | ${terraform_modules} | ${terraform_total} | ${terraform_native_files} | ${terraform_avm_files} | ${terraform_native_pct}% | ${terraform_avm_pct}% |" +echo "| **Total** | **${combined_modules}** | **${combined_total}** | **${combined_native_files}** | **${combined_avm_files}** | **${combined_native_pct}%** | **${combined_avm_pct}%** |" echo echo echo "BICEP PATTERN MODULE TESTS (File-based Analysis):" -echo "- Pattern modules: $bicep_modules" -echo "- Test files analyzed: $bicep_total" -echo "- Test files with native resources: $bicep_native_files/$bicep_total (${bicep_native_pct}%)" -echo "- Test files with AVM modules: $bicep_avm_files/$bicep_total (${bicep_avm_pct}%)" +echo "- Pattern modules: ${bicep_modules}" +echo "- Test files analyzed: ${bicep_total}" +echo "- Test files with native resources: ${bicep_native_files}/${bicep_total} (${bicep_native_pct}%)" +echo "- Test files with AVM modules: ${bicep_avm_files}/${bicep_total} (${bicep_avm_pct}%)" echo echo "TERRAFORM PATTERN MODULE EXAMPLES (File-based Analysis):" -echo "- Pattern modules: $terraform_modules" -echo "- Example files analyzed: $terraform_total" -echo "- Example files with native resources: $terraform_native_files/$terraform_total (${terraform_native_pct}%)" -echo "- Example files with AVM modules: $terraform_avm_files/$terraform_total (${terraform_avm_pct}%)" +echo "- Pattern modules: ${terraform_modules}" +echo "- Example files analyzed: ${terraform_total}" +echo "- Example files with native resources: ${terraform_native_files}/${terraform_total} (${terraform_native_pct}%)" +echo "- Example files with AVM modules: ${terraform_avm_files}/${terraform_total} (${terraform_avm_pct}%)" echo # Calculate overall statistics @@ -99,36 +132,48 @@ total_files=$((bicep_total + terraform_total)) total_native_files=$((bicep_native_files + terraform_native_files)) total_avm_files=$((bicep_avm_files + terraform_avm_files)) -if [ $total_files -gt 0 ]; then - overall_native_pct=$(echo "scale=1; $total_native_files * 100 / $total_files" | bc -l) - overall_avm_pct=$(echo "scale=1; $total_avm_files * 100 / $total_files" | bc -l) +if [ "${total_files}" -gt 0 ]; then + overall_native_pct=$(echo "scale=1; ${total_native_files} * 100 / ${total_files}" | bc -l) + overall_avm_pct=$(echo "scale=1; ${total_avm_files} * 100 / ${total_files}" | bc -l) else overall_native_pct=0 overall_avm_pct=0 fi echo "OVERALL COMBINED ANALYSIS:" -echo "- Test/Example files with native resources: $total_native_files/$total_files (${overall_native_pct}%)" -echo "- Test/Example files with AVM modules: $total_avm_files/$total_files (${overall_avm_pct}%)" +echo "- Test/Example files with native resources: ${total_native_files}/${total_files} (${overall_native_pct}%)" +echo "- Test/Example files with AVM modules: ${total_avm_files}/${total_files} (${overall_avm_pct}%)" echo echo "4. CONCLUSION" echo "=============" -if [ $total_avm_files -gt $total_native_files ]; then - echo "❌ Pattern module tests/examples STILL predominantly use NATIVE resources rather than AVM modules" - echo " Native: $total_native_files files (${overall_native_pct}%) vs AVM: $total_avm_files files (${overall_avm_pct}%)" -elif [ $total_native_files -gt $total_avm_files ]; then - echo "❌ Pattern module tests/examples predominantly use NATIVE resources rather than AVM modules" - echo " Native: $total_native_files files (${overall_native_pct}%) vs AVM: $total_avm_files files (${overall_avm_pct}%)" +if [ "${total_avm_files}" -gt "${total_native_files}" ]; then + echo "[!] Pattern module tests/examples STILL predominantly use NATIVE resources rather than AVM modules" + echo " Native: ${total_native_files} files (${overall_native_pct}%) vs AVM: ${total_avm_files} files (${overall_avm_pct}%)" +elif [ "${total_native_files}" -gt "${total_avm_files}" ]; then + echo "[!] Pattern module tests/examples predominantly use NATIVE resources rather than AVM modules" + echo " Native: ${total_native_files} files (${overall_native_pct}%) vs AVM: ${total_avm_files} files (${overall_avm_pct}%)" else - echo "⚖️ Equal usage of native resources and AVM modules in pattern module tests/examples" + echo "[=] Equal usage of native resources and AVM modules in pattern module tests/examples" fi echo echo "Key Insights:" -echo "- Bicep: $bicep_modules pattern modules with $bicep_total test files (avg $(echo "scale=1; $bicep_total / $bicep_modules" | bc -l) files/module)" -echo "- Terraform: $terraform_modules pattern modules with $terraform_total example files (avg $(echo "scale=1; $terraform_total / $terraform_modules" | bc -l) files/module)" +if [ "${bicep_modules}" -gt 0 ]; then + bicep_avg=$(echo "scale=1; ${bicep_total} / ${bicep_modules}" | bc -l) +else + bicep_avg=0 +fi +if [ "${terraform_modules}" -gt 0 ]; then + terraform_avg=$(echo "scale=1; ${terraform_total} / ${terraform_modules}" | bc -l) +else + terraform_avg=0 +fi +echo "- Bicep: ${bicep_modules} pattern modules with ${bicep_total} test files (avg ${bicep_avg} files/module)" +echo "- Terraform: ${terraform_modules} pattern modules with ${terraform_total} example files (avg ${terraform_avg} files/module)" echo "- Note: Terraform average is skewed by modules with extensive example suites" echo "- Both ecosystems show similar pattern: ~70-74% native resource usage in tests/examples" echo "- This analysis is about TEST and EXAMPLE files, NOT the pattern modules themselves" echo "- Pattern module tests/examples are not yet fully demonstrating 'modules calling modules' approach" + +log_info "Finished." diff --git a/Bash/log.sh b/Bash/log.sh new file mode 100644 index 0000000..c88b23b --- /dev/null +++ b/Bash/log.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +# +# log.sh - minimal structured, timestamped logging helpers. +# +# Source it from a script, do not execute it directly: +# . "$(dirname -- "${BASH_SOURCE[0]}")/log.sh" +# +# Provides: log_info, log_warn, log_error (warn/error go to stderr). +# +# Author: Sebastian Gräf +# Repo: https://github.com/segraef/Scripts + +# Guard against double-sourcing. +[[ -n "${__LOG_SH_SOURCED:-}" ]] && return 0 +__LOG_SH_SOURCED=1 + +_log() { + printf '[%s] %-5s %s\n' "$(date '+%Y-%m-%d %H:%M:%S')" "$1" "${*:2}" +} + +log_info() { _log "INFO" "$@"; } +log_warn() { _log "WARN" "$@" >&2; } +log_error() { _log "ERROR" "$@" >&2; } diff --git a/CHANGELOG.md b/CHANGELOG.md index 3d94642..6fbbb4c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,10 +6,22 @@ All notable changes to this project will be documented in this file. ### Added -- Added xyz [@segraef](https://github.com/segraef) +- PowerShell scaffolding refresh: `Write-Log` as a real module (`Write-Log.psm1`), updated `_Template.ps1` (PowerShell 7, `SupportsShouldProcess`, parameter validation) and a `PSScriptAnalyzerSettings.psd1` lint/format baseline [@segraef](https://github.com/segraef) +- Bash scaffolding: `_Template.sh` and `log.sh` logging helpers [@segraef](https://github.com/segraef) +- Template Sync workflow to track [segraef/Template](https://github.com/segraef/Template) [@segraef](https://github.com/segraef) +- Repository structure section in the README [@segraef](https://github.com/segraef) + +### Changed + +- Modernised CI: `actions/checkout@v4`, `super-linter@v7`, latest PSScriptAnalyzer with shared settings, linters run on push and pull request [@segraef](https://github.com/segraef) + +### Fixed + +- Corrected `segraef/Template` references that pointed away from this repository (README badge, issue links) [@segraef](https://github.com/segraef) +- Expanded `.gitignore` (Python bytecode/venv, Node, logs); removed stray files [@segraef](https://github.com/segraef) ## [1.0.0] - 2021-06-25 ### Added -- Inititated repository [@segraef](https://github.com/segraef) \ No newline at end of file +- Initiated repository [@segraef](https://github.com/segraef) \ No newline at end of file diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md index bbb2e31..7bf19e5 100644 --- a/CODE_OF_CONDUCT.md +++ b/CODE_OF_CONDUCT.md @@ -55,7 +55,7 @@ a project may be further defined and clarified by project maintainers. ## Enforcement Instances of abusive, harassing, or otherwise unacceptable behavior may be -reported by contacting the project team at [INSERT EMAIL ADDRESS]. All +reported by contacting the project team at . All complaints will be reviewed and investigated and will result in a response that is deemed necessary and appropriate to the circumstances. The project team is obligated to maintain confidentiality with regard to the reporter of an incident. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index bad7309..508c516 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -10,8 +10,25 @@ If you find any bugs, please file an issue in the [GitHub issues][GitHubIssues] If you are taking the time to mention a problem, even a seemingly minor one, it is greatly appreciated, and a totally valid contribution to this project. Thank you! +## Conventions + +- **PowerShell** (target 7+): start new scripts from + [`PowerShell/_Template.ps1`](PowerShell/_Template.ps1). Use comment-based help, + `[CmdletBinding()]`, approved verbs, parameter validation, and + `SupportsShouldProcess` for state-changing operations. Log via + [`Write-Log.psm1`](PowerShell/Write-Log.psm1). Never hardcode tokens, + subscription IDs, or account names; take them as parameters + (`[securestring]` for secrets). +- **Bash**: start from [`Bash/_Template.sh`](Bash/_Template.sh); use + `set -euo pipefail`, quote expansions, and source + [`Bash/log.sh`](Bash/log.sh) for output. +- **Linting**: PowerShell is checked by PSScriptAnalyzer against + [`PSScriptAnalyzerSettings.psd1`](PSScriptAnalyzerSettings.psd1); Bash by + ShellCheck; everything by Super-Linter, on push and pull request. Run + `Invoke-ScriptAnalyzer` with the settings file and `shellcheck -x` locally + before opening a PR. + -[GitHubIssues]: -[Contributing]: CONTRIBUTING.md +[GitHubIssues]: diff --git a/LICENSE b/LICENSE index e523f61..6b040c0 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2021 Sebastian Gräf +Copyright (c) 2021-2026 Sebastian Gräf Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/PSScriptAnalyzerSettings.psd1 b/PSScriptAnalyzerSettings.psd1 new file mode 100644 index 0000000..8392d42 --- /dev/null +++ b/PSScriptAnalyzerSettings.psd1 @@ -0,0 +1,48 @@ +@{ + # PSScriptAnalyzer configuration for this repository. + # Invoke with: Invoke-ScriptAnalyzer -Path . -Recurse -Settings ./PSScriptAnalyzerSettings.psd1 + # Formatting can be applied with: Invoke-Formatter -Settings ./PSScriptAnalyzerSettings.psd1 + + # Fail CI on errors and warnings; informational findings stay advisory. + Severity = @('Error', 'Warning') + + IncludeDefaultRules = $true + + # Rules that are noisy for a personal automation repo, or handled elsewhere. + ExcludeRules = @( + 'PSReviewUnusedParameter', + # Files are UTF-8 without BOM (correct for PowerShell 7 / cross-platform). + # This rule only matters for Windows PowerShell 5.1. + 'PSUseBOMForUnicodeEncodedFile' + ) + + Rules = @{ + # Enforce the house style (4-space indent, OTBS braces, aligned assignments). + PSPlaceOpenBrace = @{ + Enable = $true + OnSameLine = $true + NewLineAfter = $true + } + PSPlaceCloseBrace = @{ + Enable = $true + NewLineAfter = $true + } + PSUseConsistentIndentation = @{ + Enable = $true + Kind = 'space' + IndentationSize = 4 + } + PSUseConsistentWhitespace = @{ + Enable = $true + } + PSUseCorrectCasing = @{ + Enable = $true + } + # Every script/function must carry comment-based help. + PSProvideCommentHelp = @{ + Enable = $true + ExportedOnly = $false + Placement = 'before' + } + } +} diff --git a/PowerShell/Activity-Simulator.ps1 b/PowerShell/Activity-Simulator.ps1 index b864eb8..9e02c3d 100644 --- a/PowerShell/Activity-Simulator.ps1 +++ b/PowerShell/Activity-Simulator.ps1 @@ -1,70 +1,88 @@ -#Requires -Version 5.1 +#Requires -Version 7.0 <# .SYNOPSIS - Simulates Mouse and Keyboard Activity to avoid Screensaver coming up. + Simulates mouse and keyboard activity to stop the screensaver from starting. .DESCRIPTION - Simulates Mouse and Keyboard Activity to avoid Screensaver coming up. + Sends a periodic keystroke and nudges the cursor for the requested number of + minutes so the session stays active and the screensaver or lock timeout does + not trigger. A progress bar shows the remaining time. If no duration is given + the script prompts for one. .PARAMETER Minutes - Commit minutes for simulating activity. If no string given you will be asked. + Number of minutes to simulate activity for. If omitted, the script prompts + for a value. -.PARAMETER Verbose - Run in Verbose Mode. +.INPUTS + None. This script does not accept pipeline input. + +.OUTPUTS + None. Status is written to the host and progress streams. .EXAMPLE - PS C:\> Avtivity-Simulator.ps1 -Minutes 60 + ./Activity-Simulator.ps1 -Minutes 60 + Keeps the session active for 60 minutes. .LINK - https://graef.io + https://graef.io .NOTES - Author: Sebastian Gräf - Email: sebastian@graef.io - Date: September 9, 2017 + Author: Sebastian Gräf + Repo: https://github.com/segraef/Scripts #> -[Cmdletbinding()] -Param ( - [Parameter(Mandatory = $false)] - [string]$Minutes +#region Parameters +[CmdletBinding()] +param +( + [Parameter()] + [string]$Minutes ) +#endregion + +#region Execution +begin { + Set-StrictMode -Version Latest + $ErrorActionPreference = 'Stop' -Begin { - Write-Verbose " [$($MyInvocation.InvocationName)] :: Start Process" + Import-Module "$PSScriptRoot/Write-Log.psm1" -Force + + Write-Verbose " [$($MyInvocation.InvocationName)] :: Start Process" } -Process { - Add-Type -AssemblyName System.Windows.Forms - $shell = New-Object -com "Wscript.Shell" - - $pshost = Get-Host - $pswindow = $pshost.ui.rawui - $pswindow.windowtitle = 'Activity-Simulator' - - if (!$minutes) { - $Minutes = Read-Host -Prompt "Enter minutes for simulating activity" - } - - for ($i = 0; $i -lt $Minutes; $i++) { - $start = (Get-Date -Format HH:mm:ss) - $timeleft = $Minutes - $i - Clear-Host - Write-Output "Start: $start" - $shell.sendkeys(' ') - for ($j = 0; $j -lt 6; $j++) { - for ($k = 0; $k -lt 10; $k++) { - Write-Progress -Activity 'Simulating activity ...' -PercentComplete ($k * 10) -Status "Please wait $timeleft Minutes." - Start-Sleep -Seconds 1 - } - } - $Pos = [System.Windows.Forms.Cursor]::Position - $x = ($pos.X % 500) + 1 - $y = ($pos.Y % 500) + 1 - [System.Windows.Forms.Cursor]::Position = New-Object System.Drawing.Point($x, $y) - } +process { + Add-Type -AssemblyName System.Windows.Forms + $shell = New-Object -ComObject 'Wscript.Shell' + + $pshost = Get-Host + $pswindow = $pshost.ui.rawui + $pswindow.windowtitle = 'Activity-Simulator' + + if (!$Minutes) { + $Minutes = Read-Host -Prompt 'Enter minutes for simulating activity' + } + + for ($i = 0; $i -lt $Minutes; $i++) { + $start = (Get-Date -Format HH:mm:ss) + $timeleft = $Minutes - $i + Clear-Host + Write-Log "Start: $start" + $shell.sendkeys(' ') + for ($j = 0; $j -lt 6; $j++) { + for ($k = 0; $k -lt 10; $k++) { + Write-Progress -Activity 'Simulating activity ...' -PercentComplete ($k * 10) -Status "Please wait $timeleft Minutes." + Start-Sleep -Seconds 1 + } + } + $pos = [System.Windows.Forms.Cursor]::Position + $x = ($pos.X % 500) + 1 + $y = ($pos.Y % 500) + 1 + [System.Windows.Forms.Cursor]::Position = New-Object System.Drawing.Point($x, $y) + } } -End { - Write-Verbose " [$($MyInvocation.InvocationName)] :: End Process" + +end { + Write-Verbose " [$($MyInvocation.InvocationName)] :: End Process" } +#endregion diff --git a/PowerShell/Export-AzDOBuildReleaseDefinitions.ps1 b/PowerShell/Export-AzDOBuildReleaseDefinitions.ps1 index bc1a2a1..d92b987 100644 --- a/PowerShell/Export-AzDOBuildReleaseDefinitions.ps1 +++ b/PowerShell/Export-AzDOBuildReleaseDefinitions.ps1 @@ -1,149 +1,268 @@ -<# - .SYNOPSIS - Exports build and release definitions from a source Azure DevOps project and imports these into a destination project. - If no destination project give it will only export build and release defintions and save it as JSON. +#Requires -Version 7.0 - .DESCRIPTION - The script will create two folders with all exported build and release definitions as JSON. - Once a destination organization and project are given, the script will import the exported - build and release defintions into the target project. +<# +.SYNOPSIS + Exports build and release definitions from a source Azure DevOps project and optionally imports them into a destination project. - Required Modules (will be installed if not present): VSTeam +.DESCRIPTION + Connects to a source Azure DevOps organisation/project with VSTeam and exports every + build and release definition to JSON, written into two folders named + "..BuildDefinitions" and "..ReleaseDefinitions". - .PARAMETER sourceAccount - Azure DevOps source account/organization. + When a destination account and project are also supplied, the exported JSON files are + imported into the destination project. If no destination is given, the script only + exports and saves the definitions as JSON. - .PARAMETER sourceProject - Azure DevOps source project. + Required Modules (installed if not present): VSTeam. - .PARAMETER sourcePersonalAccessToken - Azure DevOps source Personal Access Token. +.PARAMETER sourceAccount + Azure DevOps source account/organisation. - .PARAMETER sourcePersonalAccessToken - Azure DevOps source Secure Access Token. +.PARAMETER sourceProject + Azure DevOps source project. - .PARAMETER destinationAccount - Azure DevOps destination account/organization. +.PARAMETER sourcePersonalAccessToken + Azure DevOps source Personal Access Token (plain string, used with -PersonalAccessToken). - .PARAMETER destinationProject - Azure DevOps destination account/organization. +.PARAMETER sourceSecureAccessToken + Azure DevOps source Secure Access Token (used with -SecurePersonalAccessToken when no plain PAT is supplied). - .PARAMETER destinationPersonalAccessToken - Azure DevOps destination Personal Access Token. +.PARAMETER destinationAccount + Azure DevOps destination account/organisation. - .PARAMETER destinationSecureAccessToken - Azure DevOps destination Secure Access Token. +.PARAMETER destinationProject + Azure DevOps destination project. - .NOTES - Version: 1.0 - Author: Sebastian Gräf - Email: sebastian@graef.io - Creation Date: 08/13/2019 - Purpose/Change: Initial script development -#> +.PARAMETER destinationPersonalAccessToken + Azure DevOps destination Personal Access Token (plain string, used with -PersonalAccessToken). + +.PARAMETER destinationSecureAccessToken + Azure DevOps destination Secure Access Token (used with -SecurePersonalAccessToken when no plain PAT is supplied). + +.INPUTS + None. -#region Parameters +.OUTPUTS + JSON files for each build and release definition, written to two folders in the current directory. -param ( - [Parameter(Mandatory = $true)] +.EXAMPLE + ./Export-AzDOBuildReleaseDefinitions.ps1 -sourceAccount 'contoso' -sourceProject 'Web' -sourcePersonalAccessToken 'pat' + Exports all build and release definitions from the Web project to JSON. + +.EXAMPLE + ./Export-AzDOBuildReleaseDefinitions.ps1 -sourceAccount 'contoso' -sourceProject 'Web' -sourcePersonalAccessToken 'pat' -destinationAccount 'fabrikam' -destinationProject 'Web' -destinationPersonalAccessToken 'pat2' + Exports definitions from the source project and imports them into the destination project. + +.NOTES + Author: Sebastian Gräf + Repo: https://github.com/segraef/Scripts + Version history is tracked in git, not in this header. +#> + +[CmdletBinding(SupportsShouldProcess)] +param +( + [Parameter(Mandatory)] + [ValidateNotNullOrEmpty()] [string]$sourceAccount, - [Parameter(Mandatory = $true)] + + [Parameter(Mandatory)] + [ValidateNotNullOrEmpty()] [string]$sourceProject, - [Parameter(Mandatory = $true)] + + [Parameter(Mandatory)] + [ValidateNotNullOrEmpty()] [string]$sourcePersonalAccessToken, - [Parameter(Mandatory = $false)] + + [Parameter()] [securestring]$sourceSecureAccessToken, - [Parameter(Mandatory = $False)] + + [Parameter()] [string]$destinationAccount, - [Parameter(Mandatory = $False)] + + [Parameter()] [string]$destinationProject, - [Parameter(Mandatory = $False)] + + [Parameter()] [string]$destinationPersonalAccessToken, - [Parameter(Mandatory = $False)] + + [Parameter()] [securestring]$destinationSecureAccessToken ) +#region Initialisation +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +Import-Module "$PSScriptRoot/Write-Log.psm1" -Force #endregion -#region Initialisations +#region Functions +function Initialize-RequiredModule { + <# + .SYNOPSIS + Ensure a PowerShell module is available, installing it from the gallery if required. + + .DESCRIPTION + Imports the named module if it is already available, otherwise installs it for the + current user from the PowerShell Gallery and then imports it. -$ErrorActionPreference = "Stop" -$VerbosePreference = "Continue" + .PARAMETER Name + The module name to ensure is loaded. -Import-Module ..\Load-Module.ps1 + .EXAMPLE + Initialize-RequiredModule -Name 'VSTeam' + Imports VSTeam, installing it first if it is not present. + #> + [CmdletBinding(SupportsShouldProcess)] + param + ( + [Parameter(Mandatory)] + [ValidateNotNullOrEmpty()] + [string]$Name + ) -#endregion + if (Get-Module -Name $Name) { + Write-Log "Module '$Name' is already imported." + return + } -#region Declarations -#endregion + try { + if (-not (Get-Module -ListAvailable -Name $Name)) { + if ($PSCmdlet.ShouldProcess($Name, 'Install module')) { + Write-Log "Installing module '$Name' from the gallery." + Install-Module -Name $Name -Force -Scope CurrentUser + } + } -#region Functions + Import-Module -Name $Name + Write-Log "Module '$Name' imported." + } + catch { + Write-Log -Message "Failed to ensure module '$Name'." -ErrorRecord $_ + throw + } +} #endregion #region Execution +Write-Log "Executing $($MyInvocation.MyCommand.Name)." + +Initialize-RequiredModule -Name 'VSTeam' -Load-Module VSTeam +$buildDefinitionDirectory = $null +$releaseDefinitionDirectory = $null if ($sourceAccount -and $sourceProject) { - # Set the source project + # Set the source project. if ($sourcePersonalAccessToken) { Set-VSTeamAccount -Account $sourceAccount -PersonalAccessToken $sourcePersonalAccessToken } - elseif ($sourcePersonalAccessToken) { + elseif ($sourceSecureAccessToken) { Set-VSTeamAccount -Account $sourceAccount -SecurePersonalAccessToken $sourceSecureAccessToken } else { - Write-Output "Exiting script since no token given" + Write-Log 'Exiting script since no source token given.' -Level Warning + return } - # Get all release definitions - $releaseDefinitions = Get-VSTeamReleaseDefinition -ProjectName $sourceProject + try { + # Get all release definitions. + $releaseDefinitions = Get-VSTeamReleaseDefinition -ProjectName $sourceProject - # Get all build definitions - $buildDefinitions = Get-VSTeamBuildDefinition -ProjectName $sourceProject + # Get all build definitions. + $buildDefinitions = Get-VSTeamBuildDefinition -ProjectName $sourceProject + } + catch { + Write-Log -Message "Failed to read definitions from project '$sourceProject'." -ErrorRecord $_ + throw + } - # Create definition folders - $buildDefinitionDirectory = New-Item "$sourceAccount.$sourceProject.BuildDefinitions" -ItemType Directory -Force - $releaseDefinitionDirectory = New-Item "$sourceAccount.$sourceProject.ReleaseDefinitions" -ItemType Directory -Force + # Create definition folders. + if ($PSCmdlet.ShouldProcess("$sourceAccount.$sourceProject", 'Create export folders')) { + try { + $buildDefinitionDirectory = New-Item "$sourceAccount.$sourceProject.BuildDefinitions" -ItemType Directory -Force + $releaseDefinitionDirectory = New-Item "$sourceAccount.$sourceProject.ReleaseDefinitions" -ItemType Directory -Force + } + catch { + Write-Log -Message 'Failed to create export folders.' -ErrorRecord $_ + throw + } + } - # Export build defintions + # Export build definitions. foreach ($buildDefinition in $buildDefinitions) { - $fileName = $buildDefinitionDirectory.FullName + "\" + $buildDefinition.Name + ".json" - Get-VSTeamBuildDefinition -ProjectName $sourceProject -Id $buildDefinition.ID -json | Out-File $fileName + $fileName = Join-Path $buildDefinitionDirectory.FullName "$($buildDefinition.Name).json" + if ($PSCmdlet.ShouldProcess($fileName, 'Export build definition')) { + try { + Get-VSTeamBuildDefinition -ProjectName $sourceProject -Id $buildDefinition.ID -json | Out-File $fileName + } + catch { + Write-Log -Message "Failed to export build definition '$($buildDefinition.Name)'." -ErrorRecord $_ + throw + } + } } - Write-Output "Your build definitions can be found here: $buildDefinitionDirectory" + Write-Log "Your build definitions can be found here: $buildDefinitionDirectory" - # Export release defintions + # Export release definitions. foreach ($releaseDefinition in $releaseDefinitions) { - $fileName = $releaseDefinitionDirectory.FullName + "\" + $releaseDefinition.Name + ".json" - Get-VSTeamReleaseDefinition -ProjectName $sourceProject -Id $releaseDefinition.ID -json | Out-File $fileName + $fileName = Join-Path $releaseDefinitionDirectory.FullName "$($releaseDefinition.Name).json" + if ($PSCmdlet.ShouldProcess($fileName, 'Export release definition')) { + try { + Get-VSTeamReleaseDefinition -ProjectName $sourceProject -Id $releaseDefinition.ID -json | Out-File $fileName + } + catch { + Write-Log -Message "Failed to export release definition '$($releaseDefinition.Name)'." -ErrorRecord $_ + throw + } + } } - Write-Output "Your release definitions can be found here: $releaseDefinitionDirectory" + Write-Log "Your release definitions can be found here: $releaseDefinitionDirectory" } if ($destinationAccount -and $destinationProject) { - # Set the destination project + # Set the destination project. if ($destinationPersonalAccessToken) { Set-VSTeamAccount -Account $destinationAccount -PersonalAccessToken $destinationPersonalAccessToken } elseif ($destinationSecureAccessToken) { - Set-VSTeamAccount -Account $sourceAccount -SecurePersonalAccessToken $destinationSecureAccessToken + Set-VSTeamAccount -Account $destinationAccount -SecurePersonalAccessToken $destinationSecureAccessToken } else { - Write-Output "Exiting script since no token given" + Write-Log 'Exiting script since no destination token given.' -Level Warning + return } - # Import release defintions - $releaseDefinitions = Get-ChildItem $releaseDefinitionDirectory.FullName - foreach ($releaseDefinition in $releaseDefinitions) { - $fileName = $releaseDefinition.FullName - Add-VSTeamReleaseDefinition -ProjectName $destinationProject -inFile $fileName + if (-not $releaseDefinitionDirectory -or -not $buildDefinitionDirectory) { + Write-Log 'No exported definitions available to import; run the export step first.' -Level Warning + return } - # Import build defintions - $buildDefinitions = Get-ChildItem $buildDefinitionDirectory.FullName - foreach ($buildDefinition in $buildDefinitions) { - $fileName = $buildDefinition.FullName - Add-VSTeamReleaseDefinition -ProjectName $destinationProject -inFile $fileName + try { + # Import release definitions. + $releaseDefinitions = Get-ChildItem $releaseDefinitionDirectory.FullName + foreach ($releaseDefinition in $releaseDefinitions) { + $fileName = $releaseDefinition.FullName + if ($PSCmdlet.ShouldProcess($fileName, 'Import release definition')) { + Add-VSTeamReleaseDefinition -ProjectName $destinationProject -inFile $fileName + } + } + + # Import build definitions. + $buildDefinitions = Get-ChildItem $buildDefinitionDirectory.FullName + foreach ($buildDefinition in $buildDefinitions) { + $fileName = $buildDefinition.FullName + if ($PSCmdlet.ShouldProcess($fileName, 'Import build definition')) { + Add-VSTeamBuildDefinition -ProjectName $destinationProject -inFile $fileName + } + } + } + catch { + Write-Log -Message "Failed to import definitions into project '$destinationProject'." -ErrorRecord $_ + throw } } + +Write-Log "Finished $($MyInvocation.MyCommand.Name)." +#endregion diff --git a/PowerShell/Get-DevOpsPrivateRepoFile.ps1 b/PowerShell/Get-DevOpsPrivateRepoFile.ps1 index 0d44289..cccf061 100644 --- a/PowerShell/Get-DevOpsPrivateRepoFile.ps1 +++ b/PowerShell/Get-DevOpsPrivateRepoFile.ps1 @@ -1,111 +1,237 @@ -#Requires -Version 5.1 +#Requires -Version 7.0 <# .SYNOPSIS - + Retrieves the content of a single file from a private Azure DevOps Git repository. .DESCRIPTION - + Calls the Azure DevOps Git Items REST API to download the text content of one + file from a private repository, authenticating with a Personal Access Token (PAT) + over HTTP Basic auth. The file content is written to the output stream so it can + be captured, piped, or redirected to disk. -.PARAMETER - + Prerequisites: + - An Azure DevOps organisation reachable at https://.visualstudio.com. + - A PAT with at least Code (Read) scope on the target project/repository. + +.PARAMETER DevOpsAccountName + The Azure DevOps organisation (account) name. Used to build the base URL + https://.visualstudio.com. + +.PARAMETER DevOpsTeamProjectName + The team project that contains the target repository. + +.PARAMETER FileRepo + The name (or id) of the Git repository to read the file from. + +.PARAMETER DevOpsPAT + The Azure DevOps Personal Access Token used to authenticate, supplied as a + SecureString. Requires Code (Read) scope. + +.PARAMETER User + The user name portion of the Basic auth pair. Azure DevOps PAT auth ignores the + user name, so this is normally left empty. + +.PARAMETER FileRepoBranch + The branch to read the file from. Defaults to 'master'. + +.PARAMETER FilePath + The repository-relative path of the file to retrieve. + +.PARAMETER ApiVersion + The Azure DevOps REST API version to target. Defaults to '4.1'. .INPUTS - + None. This script does not accept pipeline input. .OUTPUTS - .log> + System.String. The text content of the requested file is written to the output + stream. -.NOTES - Version: 1.0 - Author: - Creation Date: - Purpose/Change: Initial script development +.EXAMPLE + $pat = Read-Host -AsSecureString + ./Get-DevOpsPrivateRepoFile.ps1 -DevOpsAccountName 'contoso' -DevOpsTeamProjectName 'Platform' -FileRepo 'Platform' -DevOpsPAT $pat -FilePath 'Scripts/PowerShell/Deploy.ps1' + Downloads Deploy.ps1 from the master branch of the Platform repository and writes + its content to the output stream. .EXAMPLE - + ./Get-DevOpsPrivateRepoFile.ps1 -DevOpsAccountName 'contoso' -DevOpsTeamProjectName 'Platform' -FileRepo 'Platform' -DevOpsPAT $pat -FilePath 'README.md' -FileRepoBranch 'main' > README.md + Downloads README.md from the main branch and saves it to a local file. + +.NOTES + Author: Sebastian Gräf + Repo: https://github.com/segraef/Scripts #> #region Parameters - [CmdletBinding()] param ( - [Parameter()] - [String]$devOpsAccountName = 'segraef', - [Parameter()] - [String]$devOpsTeamProjectName = 'Oahu', - [Parameter()] - [String]$devOpsPAT = 'xxx', - [Parameter()] - [String]$devOpsBaseUrl = 'https://' + $devOpsAccountName + '.visualstudio.com', - [Parameter()] - [String]$FileRepo = 'Oahu', - [Parameter()] - [String]$FileRepoBranch = 'master', - [Parameter()] - [String]$FilePath = 'Scripts/PowerShell/123.ps1', - [Parameter()] - [String]$User = '' -) + [Parameter(Mandatory)] + [ValidateNotNullOrEmpty()] + [string]$DevOpsAccountName, -#endregion + [Parameter(Mandatory)] + [ValidateNotNullOrEmpty()] + [string]$DevOpsTeamProjectName, -#region Initialisations + [Parameter(Mandatory)] + [ValidateNotNullOrEmpty()] + [string]$FileRepo, -$ErrorActionPreference = "Continue" -$VerbosePreference = "Continue" + [Parameter(Mandatory)] + [ValidateNotNull()] + [securestring]$DevOpsPAT, -# Dot Source required Function Libraries -Import-Module ..\Write-Log.ps1 + [Parameter()] + [string]$User = '', + [Parameter()] + [ValidateNotNullOrEmpty()] + [string]$FileRepoBranch = 'master', + + [Parameter(Mandatory)] + [ValidateNotNullOrEmpty()] + [string]$FilePath, + + [Parameter()] + [ValidateNotNullOrEmpty()] + [string]$ApiVersion = '4.1' +) #endregion -#region Declarations +#region Initialisation +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +Import-Module "$PSScriptRoot/Write-Log.psm1" -Force #endregion #region Functions +function Get-DevOpsRepoFileContent { + <# + .SYNOPSIS + Downloads the content of one file from a private Azure DevOps Git repository. + + .DESCRIPTION + Builds the Azure DevOps Git Items REST API URI for the requested file and + retrieves its content using PAT-based Basic authentication. + + .PARAMETER DevOpsAccountName + The Azure DevOps organisation (account) name. + + .PARAMETER DevOpsTeamProjectName + The team project that contains the target repository. + + .PARAMETER FileRepo + The Git repository to read the file from. + + .PARAMETER DevOpsPAT + The Azure DevOps Personal Access Token, supplied as a SecureString. -function FunctionName { - Param() + .PARAMETER User + The user name portion of the Basic auth pair (normally empty for PAT auth). - begin { - Write-Log "Let's start !" - } + .PARAMETER FileRepoBranch + The branch to read the file from. + + .PARAMETER FilePath + The repository-relative path of the file to retrieve. + + .PARAMETER ApiVersion + The Azure DevOps REST API version to target. + + .OUTPUTS + System.String. The text content of the requested file. + + .EXAMPLE + Get-DevOpsRepoFileContent -DevOpsAccountName 'contoso' -DevOpsTeamProjectName 'Platform' -FileRepo 'Platform' -DevOpsPAT $pat -FilePath 'README.md' + Returns the content of README.md from the master branch. + #> + [CmdletBinding()] + [OutputType([string])] + param + ( + [Parameter(Mandatory)] + [ValidateNotNullOrEmpty()] + [string]$DevOpsAccountName, + + [Parameter(Mandatory)] + [ValidateNotNullOrEmpty()] + [string]$DevOpsTeamProjectName, + + [Parameter(Mandatory)] + [ValidateNotNullOrEmpty()] + [string]$FileRepo, + + [Parameter(Mandatory)] + [ValidateNotNull()] + [securestring]$DevOpsPAT, + + [Parameter()] + [string]$User = '', + + [Parameter()] + [ValidateNotNullOrEmpty()] + [string]$FileRepoBranch = 'master', + + [Parameter(Mandatory)] + [ValidateNotNullOrEmpty()] + [string]$FilePath, + + [Parameter()] + [ValidateNotNullOrEmpty()] + [string]$ApiVersion = '4.1' + ) + + $devOpsBaseUrl = "https://$DevOpsAccountName.visualstudio.com" + $plainPat = [System.Net.NetworkCredential]::new('', $DevOpsPAT).Password - process { try { - Write-Output "Hello Template !" - } + $base64AuthInfo = [Convert]::ToBase64String([Text.Encoding]::ASCII.GetBytes(('{0}:{1}' -f $User, $plainPat))) + $devOpsAuthHeader = @{ Authorization = ('Basic {0}' -f $base64AuthInfo) } + + $itemsPath = "/$DevOpsTeamProjectName/_apis/git/repositories/$FileRepo/items" + $queryParts = @( + "path=$FilePath" + '$format=json' + 'includeContent=true' + "versionDescriptor.version=$FileRepoBranch" + 'versionDescriptor.versionType=branch' + "api-version=$ApiVersion" + ) + $uri = $devOpsBaseUrl + $itemsPath + '?' + ($queryParts -join '&') + + Write-Log "Requesting '$FilePath' from repository '$FileRepo' (branch '$FileRepoBranch')." + $file = Invoke-RestMethod -Method Get -ContentType 'application/json' -Uri $uri -Headers $devOpsAuthHeader + + return $file.content + } catch { - Write-Output $_ - Write-Log $_ -Warning + Write-Log -Message "Failed to retrieve '$FilePath' from repository '$FileRepo'." -ErrorRecord $_ + throw } - } - - end { - if ($?) { - Write-Log "Completed successfully !" + finally { + $plainPat = $null } - } } - #endregion #region Execution +Write-Log "Executing $($MyInvocation.MyCommand.Name)." -Write-Log "Executing $($MyInvocation.MyCommand.Name)" - -$base64AuthInfo = [Convert]::ToBase64String([Text.Encoding]::ASCII.GetBytes(("{0}:{1}" -f $User,$devOpsPAT))); -$devOpsAuthHeader = @{Authorization=("Basic {0}" -f $base64AuthInfo)}; - -$Uri = $devOpsBaseUrl + '/' + $devOpsTeamProjectName + '/_apis/git/repositories/' + $FileRepo + '/items?path=' + $FilePath + '&$format=json&includeContent=true&versionDescriptor.version=' + $FileRepoBranch + '&versionDescriptor.versionType=branch&api-version=4.1' - -$File = Invoke-RestMethod -Method Get -ContentType application/json -Uri $Uri -Headers $devOpsAuthHeader - -Write-Output $File.content +$content = Get-DevOpsRepoFileContent ` + -DevOpsAccountName $DevOpsAccountName ` + -DevOpsTeamProjectName $DevOpsTeamProjectName ` + -FileRepo $FileRepo ` + -DevOpsPAT $DevOpsPAT ` + -User $User ` + -FileRepoBranch $FileRepoBranch ` + -FilePath $FilePath ` + -ApiVersion $ApiVersion -Write-Log "Finished executing $($MyInvocation.MyCommand.Name)" +Write-Output $content +Write-Log "Finished $($MyInvocation.MyCommand.Name)." #endregion diff --git a/PowerShell/Get-GitHubRateLimit.ps1 b/PowerShell/Get-GitHubRateLimit.ps1 new file mode 100644 index 0000000..d581d43 --- /dev/null +++ b/PowerShell/Get-GitHubRateLimit.ps1 @@ -0,0 +1,465 @@ +#Requires -Version 7.0 + +<# +.SYNOPSIS + Displays GitHub API rate-limit buckets using GitHub CLI. + +.DESCRIPTION + Queries GitHub's /rate_limit endpoint through GitHub CLI and renders the + response as either formatted JSON or a terminal-friendly table sorted by the + most constrained buckets first. + + This PowerShell version only requires GitHub CLI. Unlike the Bash version, it + does not require jq because JSON parsing is handled natively by PowerShell. + + Requirements: + - GitHub CLI: gh + - An authenticated gh session + + Setup: + Install GitHub CLI, then authenticate once: + + gh auth login + + Examples: + macOS: brew install gh + Windows: winget install --id GitHub.cli + Linux: https://cli.github.com/ for distro-specific packages + + This script polls GitHub's /rate_limit endpoint, which is exempt from rate + limiting, so watch mode is safe to use. + +.PARAMETER Watch + Refresh continuously until interrupted. + +.PARAMETER Interval + Refresh interval in seconds for watch mode. Default is 10. + +.PARAMETER Json + Print the raw API response as formatted JSON. + +.PARAMETER Quiet + Only show buckets whose remaining value is below the limit and whose limit is + greater than zero. + +.PARAMETER Help + Show usage information. + +.INPUTS + None. This script does not accept pipeline input. + +.OUTPUTS + System.String. Renders the rate-limit table, formatted JSON, or usage text. + +.EXAMPLE + ./Get-GitHubRateLimit.ps1 + Print a one-off snapshot of all rate-limit buckets. + +.EXAMPLE + ./Get-GitHubRateLimit.ps1 -Watch -Interval 60 + Refresh the table every 60 seconds until interrupted. + +.EXAMPLE + ./Get-GitHubRateLimit.ps1 -Quiet + Show only buckets with limit greater than zero and remaining below limit. + +.EXAMPLE + ./Get-GitHubRateLimit.ps1 -Json + Print the raw API response as formatted JSON. + +.NOTES + Author: Sebastian Gräf + Repo: https://github.com/segraef/Scripts + Version history is tracked in git, not in this header. +#> + +[CmdletBinding()] +param +( + [Alias('w')] + [switch]$Watch, + + [Alias('i')] + [ValidateRange(1, 86400)] + [int]$Interval = 10, + + [Alias('j')] + [switch]$Json, + + [Alias('q')] + [switch]$Quiet, + + [Alias('h')] + [switch]$Help +) + +#region Initialisation +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +Import-Module "$PSScriptRoot/Write-Log.psm1" -Force +#endregion + +#region Functions +function Show-Usage { + <# + .SYNOPSIS + Print command-line usage information for this script. + + .DESCRIPTION + Emits a human-readable summary of switches, short flags, requirements and + examples to the output stream. + + .EXAMPLE + Show-Usage + Print the usage banner. + #> + [CmdletBinding()] + param () + + Write-Output @' +Get-GitHubRateLimit - GitHub API rate-limit monitor + +Usage: + Get-GitHubRateLimit.ps1 # snapshot + Get-GitHubRateLimit.ps1 -Watch # watch mode (refresh every 10s) + Get-GitHubRateLimit.ps1 -Watch -Interval 5 + Get-GitHubRateLimit.ps1 -Json # raw JSON + Get-GitHubRateLimit.ps1 -Quiet # only buckets with limit > 0 and remaining < limit + Get-GitHubRateLimit.ps1 -Help + +Short flags: + -w -i 5 -j -q -h + +Requirements: + - gh (GitHub CLI) + - an authenticated gh session + +Setup: + 1. Install GitHub CLI. + macOS: brew install gh + Windows: winget install --id GitHub.cli + Linux: https://cli.github.com/ + 2. Authenticate once: + gh auth login + +Notes: + - This PowerShell version does not require jq. + - It uses gh for API calls and auth context. + - /rate_limit itself is exempt from rate limiting, so polling it is safe. + +Examples: + Get-GitHubRateLimit.ps1 + Get-GitHubRateLimit.ps1 -Quiet + Get-GitHubRateLimit.ps1 -Watch -Interval 60 + Get-GitHubRateLimit.ps1 -Json +'@ +} + +function Test-CommandExists { + <# + .SYNOPSIS + Test whether a command is available on PATH. + + .DESCRIPTION + Returns $true when Get-Command can resolve the supplied command name, + otherwise $false. + + .PARAMETER Name + The command name to resolve (for example 'gh'). + + .EXAMPLE + Test-CommandExists -Name 'gh' + Returns $true when GitHub CLI is installed. + #> + [CmdletBinding()] + [Diagnostics.CodeAnalysis.SuppressMessageAttribute( + 'PSUseSingularNouns', '', + Justification = 'Exists is a state predicate, not a plural noun.')] + param + ( + [Parameter(Mandatory)] + [ValidateNotNullOrEmpty()] + [string]$Name + ) + + return $null -ne (Get-Command -Name $Name -ErrorAction SilentlyContinue) +} + +function Get-Style { + <# + .SYNOPSIS + Build an ANSI escape sequence for a terminal style code. + + .DESCRIPTION + Returns an ANSI escape sequence for the supplied SGR code, or an empty + string when output is redirected or the terminal is 'dumb'. + + .PARAMETER Code + The SGR (Select Graphic Rendition) code, for example '31' for red. + + .EXAMPLE + Get-Style -Code '1' + Returns the bold escape sequence when the terminal supports colour. + #> + [CmdletBinding()] + param + ( + [Parameter(Mandatory)] + [ValidateNotNullOrEmpty()] + [string]$Code + ) + + if ([Console]::IsOutputRedirected -or $env:TERM -eq 'dumb') { + return '' + } + + return [char]27 + '[' + $Code + 'm' +} + +function Get-Bar { + <# + .SYNOPSIS + Render a coloured usage bar for a rate-limit bucket. + + .DESCRIPTION + Produces a fixed-width bar whose filled portion is proportional to the + remaining quota, coloured green, yellow or red by percentage remaining. + + .PARAMETER Remaining + The remaining requests in the bucket. + + .PARAMETER Limit + The total request limit for the bucket. + + .PARAMETER Width + The bar width in characters. Default is 24. + + .EXAMPLE + Get-Bar -Remaining 50 -Limit 100 + Returns a half-filled bar. + #> + [CmdletBinding()] + param + ( + [Parameter(Mandatory)] + [int]$Remaining, + + [Parameter(Mandatory)] + [int]$Limit, + + [Parameter()] + [int]$Width = 24 + ) + + if ($Limit -le 0) { + return ' ' * $Width + } + + $filled = [math]::Floor(($Remaining * $Width) / $Limit) + if ($filled -lt 0) { + $filled = 0 + } + if ($filled -gt $Width) { + $filled = $Width + } + $empty = $Width - $filled + $pct = [math]::Floor(($Remaining * 100) / $Limit) + + $colour = $script:C_GRN + if ($pct -lt 10) { + $colour = $script:C_RED + } + elseif ($pct -lt 33) { + $colour = $script:C_YEL + } + + $filledText = if ($filled -gt 0) { '█' * $filled } else { '' } + $emptyText = if ($empty -gt 0) { '░' * $empty } else { '' } + + return "$colour$filledText$($script:C_DIM)$emptyText$($script:C_RESET)" +} + +function Get-HumanReset { + <# + .SYNOPSIS + Format a Unix reset timestamp as a human-readable countdown. + + .DESCRIPTION + Converts an absolute Unix epoch reset time into a relative string such as + 'in 4m05s', 'in 30s', or 'now' when the reset is in the past. + + .PARAMETER Target + The reset time as Unix epoch seconds. + + .EXAMPLE + Get-HumanReset -Target 1717000000 + Returns a relative countdown string for that reset time. + #> + [CmdletBinding()] + param + ( + [Parameter(Mandatory)] + [long]$Target + ) + + $now = [DateTimeOffset]::UtcNow.ToUnixTimeSeconds() + $diff = $Target - $now + if ($diff -le 0) { + return 'now' + } + + $minutes = [math]::Floor($diff / 60) + $seconds = $diff % 60 + if ($minutes -gt 0) { + return ('in {0}m{1:00}s' -f $minutes, $seconds) + } + + return ('in {0}s' -f $seconds) +} + +function Get-HostName { + <# + .SYNOPSIS + Resolve the GitHub host the current gh session is logged in to. + + .DESCRIPTION + Parses 'gh auth status' to extract the host name, falling back to + 'github.com' when it cannot be determined. + + .EXAMPLE + Get-HostName + Returns 'github.com' for a standard authenticated session. + #> + [CmdletBinding()] + param () + + try { + $status = gh auth status 2>&1 | Out-String + $match = [regex]::Match($status, 'Logged in to\s+([^\s]+)') + if ($match.Success) { + return $match.Groups[1].Value + } + } + catch { + Write-Log -Message 'Could not determine gh host; falling back to github.com.' -ErrorRecord $_ + } + + return 'github.com' +} + +function Invoke-Render { + <# + .SYNOPSIS + Fetch and render the current GitHub rate-limit state. + + .DESCRIPTION + Calls 'gh api rate_limit' and renders the response either as formatted + JSON (when -Json is set) or as a coloured table sorted by the most + constrained buckets first. Honours the script-level -Quiet switch. + + .EXAMPLE + Invoke-Render + Render a single snapshot of the rate-limit buckets. + #> + [CmdletBinding()] + param () + + $payloadText = gh api rate_limit 2>$null + if (-not $payloadText) { + throw 'gh api failed - are you authenticated? (gh auth status)' + } + + if ($Json) { + Write-Output ($payloadText | ConvertFrom-Json | ConvertTo-Json -Depth 8) + return + } + + $payload = $payloadText | ConvertFrom-Json + + $user = '?' + try { + $user = (gh api user --jq .login 2>$null).Trim() + if (-not $user) { + $user = '?' + } + } + catch { + Write-Log -Message 'Could not resolve gh user login.' -ErrorRecord $_ + } + + $hostName = Get-HostName + $now = Get-Date -Format 'HH:mm:ss' + Write-Output ("{0}{1}GitHub rate limits{2} user={3}{4}{5} host={6} {7}{8}{2}" -f $script:C_BOLD, $script:C_CYN, $script:C_RESET, $script:C_BOLD, $user, $script:C_RESET, $hostName, $script:C_DIM, $now) + Write-Output '' + + $entries = foreach ($property in $payload.resources.PSObject.Properties) { + $value = $property.Value + $pct = if ($value.limit -gt 0) { [math]::Floor(($value.remaining * 100) / $value.limit) } else { 100 } + [pscustomobject]@{ + Key = $property.Name + Remaining = [int]$value.remaining + Limit = [int]$value.limit + Used = [int]$value.used + Reset = [long]$value.reset + Pct = [int]$pct + } + } + + foreach ($entry in ($entries | Sort-Object Pct, Key)) { + if ($Quiet -and (($entry.Remaining -eq $entry.Limit) -or ($entry.Limit -eq 0))) { + continue + } + + $line = "{0}{1,-26}{2} {3,5}/{4,-5} {5} {6,3}% resets {7}" -f ` + $script:C_BOLD, $entry.Key, $script:C_RESET, $entry.Remaining, $entry.Limit, (Get-Bar -Remaining $entry.Remaining -Limit $entry.Limit -Width 24), $entry.Pct, (Get-HumanReset -Target $entry.Reset) + Write-Output $line + } +} +#endregion + +#region Execution +if ($Help) { + Show-Usage + return +} + +$script:C_RESET = Get-Style '0' +$script:C_DIM = Get-Style '2' +$script:C_BOLD = Get-Style '1' +$script:C_RED = Get-Style '31' +$script:C_YEL = Get-Style '33' +$script:C_GRN = Get-Style '32' +$script:C_CYN = Get-Style '36' + +if (-not (Test-CommandExists -Name 'gh')) { + Write-Log -Message 'gh not installed.' -Level Error + exit 1 +} + +try { + gh auth status *> $null +} +catch { + Write-Log -Message 'gh is installed but not authenticated. Run: gh auth login' -ErrorRecord $_ + exit 1 +} + +if ($Watch) { + while ($true) { + Clear-Host + try { + Invoke-Render + } + catch { + Write-Log -Message 'Failed to render rate-limit snapshot.' -ErrorRecord $_ + } + Write-Output '' + Write-Output ("{0}refresh every {1}s - ctrl-c to exit{2}" -f $script:C_DIM, $Interval, $script:C_RESET) + Start-Sleep -Seconds $Interval + } +} +else { + Invoke-Render +} +#endregion diff --git a/PowerShell/Load-Module.ps1 b/PowerShell/Load-Module.ps1 index ef720c7..8edcfbc 100644 --- a/PowerShell/Load-Module.ps1 +++ b/PowerShell/Load-Module.ps1 @@ -1,75 +1,82 @@ -#Requires -Version 5.1 +#Requires -Version 7.0 <# .SYNOPSIS - + Ensures a PowerShell module is available and imported, installing it if needed. .DESCRIPTION - + Resolves a module by name in three steps: if it is already loaded it does + nothing, if it is installed but not loaded it imports it, and if it is neither + it attempts to install it from the PowerShell Gallery (current-user scope) and + then import it. If the module cannot be found in the gallery the script logs + the failure and exits with code 1. -.PARAMETER - +.PARAMETER Module + Name of the module to load. Must be a non-empty string matching a module that + is installed locally or published to the PowerShell Gallery. .INPUTS - + None. This script does not accept pipeline input. .OUTPUTS - .log> - -.NOTES - Version: 1.0 - Author: Sebastian Gräf - Creation Date: 21/06/2019 - Purpose/Change: Initial script development + None. Progress and outcome are written to the log stream. .EXAMPLE - -#> + ./Load-Module.ps1 -Module 'Az.Accounts' + Imports Az.Accounts, installing it from the PowerShell Gallery first if it is + not already present on the machine. -#------------------------------------------------------------[Parameters]--------------------------------------------------------- +.NOTES + Author: Sebastian Gräf + Repo: https://github.com/segraef/Scripts +#> -[CmdletBinding()] -Param +#region Parameters +[CmdletBinding(SupportsShouldProcess)] +param ( - [Parameter()][string]$Module + [Parameter(Mandatory)] + [ValidateNotNullOrEmpty()] + [string]$Module ) - -#region Initialisations - -$ErrorActionPreference = "Continue" -$VerbosePreference = "Continue" - #endregion -#region Declarations -#endregion +#region Initialisation +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' -#region Functions +Import-Module "$PSScriptRoot/Write-Log.psm1" -Force #endregion #region Execution +Write-Log "Executing $($MyInvocation.MyCommand.Name)." -Write-Log "Executing $($MyInvocation.MyCommand.Name)" - -if (Get-Module | Where-Object { $_.Name -eq $Module }) { - Write-Output "Module $Module is already imported." -} -else { - if (Get-Module -ListAvailable | Where-Object { $_.Name -eq $Module }) { - Import-Module $Module -Verbose +try { + if (Get-Module | Where-Object { $_.Name -eq $Module }) { + Write-Log "Module '$Module' is already imported." } - else { - if (Find-Module -Name $Module | Where-Object { $_.Name -eq $Module }) { - Install-Module -Name $Module -Force -Verbose -Scope CurrentUser - Import-Module $Module -Verbose + elseif (Get-Module -ListAvailable | Where-Object { $_.Name -eq $Module }) { + if ($PSCmdlet.ShouldProcess($Module, 'Import module')) { + Import-Module $Module + Write-Log "Imported module '$Module'." } - else { - Write-Output "Module $Module not imported, not available and not in online gallery, exiting." - EXIT 1 + } + elseif (Find-Module -Name $Module | Where-Object { $_.Name -eq $Module }) { + if ($PSCmdlet.ShouldProcess($Module, 'Install and import module')) { + Install-Module -Name $Module -Force -Scope CurrentUser + Import-Module $Module + Write-Log "Installed and imported module '$Module'." } } + else { + Write-Log "Module '$Module' not imported, not available and not in the online gallery, exiting." -Level Warning + exit 1 + } +} +catch { + Write-Log -Message "Failed to load module '$Module'." -ErrorRecord $_ + throw } -Write-Log "Finished executing $($MyInvocation.MyCommand.Name)" - +Write-Log "Finished executing $($MyInvocation.MyCommand.Name)." #endregion diff --git a/PowerShell/New-AzPipeline.ps1 b/PowerShell/New-AzPipeline.ps1 index 762f385..6097353 100644 --- a/PowerShell/New-AzPipeline.ps1 +++ b/PowerShell/New-AzPipeline.ps1 @@ -1,18 +1,26 @@ +#Requires -Version 7.0 + <# .SYNOPSIS - Create Azure Pipelines and Build Validation Checks. + Create Azure Pipelines and Pull Request Build Validation checks. .DESCRIPTION - This script is used to create Azure Pipelines. - If this scripts is run within an Azure pipeline the environment variable AZURE_DEVOPS_EXT_PAT needs to be set with $(System.AccessToken) within your pipeline. - Since tty is not supported within a pipelune run, az devops login is using the token which is set via AZURE_DEVOPS_EXT_PAT. + Logs in to Azure DevOps with a Personal Access Token, discovers every + 'pipeline.yml' file under a source path, and creates an Azure Pipeline for + each one whose name does not already exist in the target folder. Optionally + creates a branch Build Validation policy for each new pipeline. + + When run inside an Azure Pipeline, set the AZURE_DEVOPS_EXT_PAT environment + variable to $(System.AccessToken); az devops login consumes it because tty is + not available in a pipeline run. -.REQUIREMENTS - - Azure CLI 2.13.0 - - Azure CLI extension devops 0.18.0 - - Repository for which the pipeline needs to be configured. - - The '' Build Service needs 'Edit build pipeline' permissions - Reference: https://docs.microsoft.com/en-us/azure/devops/pipelines/policies/permissions?view=azure-devops#pipeline-permissions + Prerequisites: + - Azure CLI 2.13.0 or later. + - Azure CLI 'azure-devops' extension 0.18.0 or later (auto-installed). + - A repository for which the pipeline needs to be configured. + - The '' Build Service must have 'Edit build pipeline' + permission. See + https://learn.microsoft.com/azure/devops/pipelines/policies/permissions .PARAMETER OrganizationName Required. The name of the Azure DevOps organization. @@ -24,9 +32,10 @@ Required. Repository for which the pipeline needs to be configured. .PARAMETER PAT - Required. The access token whith appropirate permissions to create Azure Pipelines. - Usually the System.AccessToken from an Azure Pipeline instance run has sufficent permissions as well. - Reference: https://docs.microsoft.com/en-us/azure/devops/pipelines/process/access-tokens?view=azure-devops&tabs=yaml#how-do-i-determine-the-job-authorization-scope-of-my-yaml-pipeline + Required. The access token with appropriate permissions to create Azure + Pipelines, supplied as a SecureString. The System.AccessToken from an Azure + Pipeline run usually has sufficient permissions. See + https://learn.microsoft.com/azure/devops/pipelines/process/access-tokens .PARAMETER BranchName Optional. Branch name for which the pipelines will be configured. @@ -36,142 +45,375 @@ Optional. Path of the folder where the pipeline needs to be created. .PARAMETER PipelineSourcePath - Optional. Path of the pipelines yaml file(s) to be used for creating Azure Pipelines. - Based on the given folder all 'pipeline.yml' files will be searched within that and created accordingly. - Default is the execution path '/.' of this script. + Optional. Path of the pipeline YAML file(s) used for creating Azure Pipelines. + All 'pipeline.yml' files under the given folder are searched and created + accordingly. Default is the current execution path. + +.PARAMETER CreateBuildValidation + Optional. Also create a Pull Request Build Validation policy for each new + pipeline. -.PARAMETER createBuildValidation - Optional. Create Pull Request Build Validation in additon. +.INPUTS + None. This script does not accept pipeline input. + +.OUTPUTS + None. Creates Azure Pipelines and (optionally) Build Validation policies as a + side effect. .EXAMPLE - New-AzPipeline -OrganizationName graef.io -ProjectName Project1 -RepositoryName Repository1 -PAT + $pat = Read-Host -AsSecureString + ./New-AzPipeline.ps1 -OrganizationName graef.io -ProjectName Project1 -RepositoryName Repository1 -PAT $pat - Create all pipelines for the project 'graef.io/Project1' using a PAT. - The Azure Pipelines will be configured to use the default branch 'main' and the given repository name. + Create all pipelines for the project 'graef.io/Project1' using a PAT. The + pipelines are configured to use the default branch 'main' and the given + repository. Each 'pipeline.yml' under the source path produces one Azure + Pipeline named after its parent folder. - Given the 'PipelineSourcePath' and the default source folder patter the script will browse all *.yml files in the - and takes the parent folder as the desired name for the Azure Pipeline name to be created. +.NOTES + Author: Sebastian Gräf + Repo: https://github.com/segraef/Scripts + Version history is tracked in git, not in this header. #> -[CmdletBinding()] -param ( - [Parameter(Mandatory = $true, HelpMessage = "Azure DevOps Organization: ")][string]$OrganizationName, - [Parameter(Mandatory = $true, HelpMessage = "Azure DevOps Project: ")][string]$ProjectName, - [Parameter(Mandatory = $true, HelpMessage = "Azure DevOps Repository: ")][string]$RepositoryName, - [Parameter(Mandatory = $true, HelpMessage = "Azure DevOps Personal Access Token: ")][string]$PAT, - [Parameter(Mandatory = $false, HelpMessage = "Azure DevOps branch: ")][string]$BranchName = "main", - [Parameter(Mandatory = $false)][string]$PipelineTargetPath, - [Parameter(Mandatory = $false)][string]$PipelineSourcePath, - [Parameter(Mandatory = $false)][bool]$CreateBuildValidation = $false +[CmdletBinding(SupportsShouldProcess)] +param +( + [Parameter(Mandatory, HelpMessage = 'Azure DevOps Organization: ')] + [ValidateNotNullOrEmpty()] + [string]$OrganizationName, + + [Parameter(Mandatory, HelpMessage = 'Azure DevOps Project: ')] + [ValidateNotNullOrEmpty()] + [string]$ProjectName, + + [Parameter(Mandatory, HelpMessage = 'Azure DevOps Repository: ')] + [ValidateNotNullOrEmpty()] + [string]$RepositoryName, + + [Parameter(Mandatory, HelpMessage = 'Azure DevOps Personal Access Token: ')] + [ValidateNotNull()] + [securestring]$PAT, + + [Parameter()] + [ValidateNotNullOrEmpty()] + [string]$BranchName = 'main', + + [Parameter()] + [string]$PipelineTargetPath, + + [Parameter()] + [string]$PipelineSourcePath, + + [Parameter()] + [switch]$CreateBuildValidation ) +#region Initialisation +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +Import-Module "$PSScriptRoot/Write-Log.psm1" -Force +#endregion + +#region Functions +function Connect-AzDevOpsCli { + <# + .SYNOPSIS + Install the Azure DevOps CLI extension and log in with a PAT. + + .DESCRIPTION + Ensures dynamic extension install is enabled, adds/upgrades the + azure-devops extension, verifies the CLI is available, logs in using the + supplied PAT via the AZURE_DEVOPS_EXT_PAT environment variable, and sets + the default organization and project. + + .PARAMETER OrganizationUrl + The full Azure DevOps organization URL (https://dev.azure.com//). + + .PARAMETER ProjectName + The Azure DevOps project name to set as the default. + + .PARAMETER PAT + The Personal Access Token as a SecureString. + + .EXAMPLE + Connect-AzDevOpsCli -OrganizationUrl 'https://dev.azure.com/graef.io/' -ProjectName Project1 -PAT $pat + #> + [CmdletBinding()] + param + ( + [Parameter(Mandatory)] + [ValidateNotNullOrEmpty()] + [string]$OrganizationUrl, + + [Parameter(Mandatory)] + [ValidateNotNullOrEmpty()] + [string]$ProjectName, + + [Parameter(Mandatory)] + [ValidateNotNull()] + [securestring]$PAT + ) + + Write-Log 'Installing Azure CLI extension devops.' + az config set extension.use_dynamic_install=yes_without_prompt # allow installing extensions without prompt + az extension add --upgrade -n azure-devops + + Write-Log 'Checking availability of Azure CLI and the Azure DevOps CLI extension.' + az | Out-Null + az devops -h | Out-Null + + Write-Log "Logging in to Azure DevOps project at $OrganizationUrl$ProjectName with a PAT." + $plainPat = [System.Net.NetworkCredential]::new('', $PAT).Password + $env:AZURE_DEVOPS_EXT_PAT = $plainPat + Write-Output $plainPat | az devops login + + Write-Log "Setting default Azure DevOps configuration to $OrganizationUrl and $ProjectName." + az devops configure --defaults organization="$OrganizationUrl" project="$ProjectName" --use-git-aliases true +} + +function Get-ExistingAzPipeline { + <# + .SYNOPSIS + List existing Azure Pipelines in the target folder. + + .DESCRIPTION + Queries Azure DevOps for the pipelines that already exist under the target + folder so they can be skipped during creation. + + .PARAMETER OrganizationUrl + The full Azure DevOps organization URL. + + .PARAMETER ProjectName + The Azure DevOps project name. + + .PARAMETER PipelineTargetPath + The folder path under which to list pipelines. + + .EXAMPLE + Get-ExistingAzPipeline -OrganizationUrl $url -ProjectName Project1 -PipelineTargetPath '/' + #> + [CmdletBinding()] + param + ( + [Parameter(Mandatory)] + [ValidateNotNullOrEmpty()] + [string]$OrganizationUrl, + + [Parameter(Mandatory)] + [ValidateNotNullOrEmpty()] + [string]$ProjectName, + + [Parameter()] + [string]$PipelineTargetPath + ) + + Write-Log "Listing all Azure Pipelines in $PipelineTargetPath." + $azurePipelines = az pipelines list --organization $OrganizationUrl --project $ProjectName --folder-path $PipelineTargetPath | + ConvertFrom-Json | + Sort-Object name + Write-Log "Found $($azurePipelines.Count) Azure Pipeline(s) in $ProjectName." + + Write-Output $azurePipelines +} + +function Get-YamlPipelineDefinition { + <# + .SYNOPSIS + Discover 'pipeline.yml' files and build pipeline definition objects. + + .DESCRIPTION + Recursively searches the source path for 'pipeline.yml' files and, for + each one, builds a PSCustomObject describing the pipeline to create + (project, repository, branch, folder, relative YAML path, and the name + derived from the parent folder). + + .PARAMETER PipelineSourcePath + The folder under which to search for 'pipeline.yml' files. Resolved + relative to the current location. + + .PARAMETER ProjectName + The Azure DevOps project name. + + .PARAMETER RepositoryName + The repository name for which the pipelines are configured. + + .PARAMETER BranchName + The branch name for which the pipelines are configured. + + .PARAMETER PipelineTargetPath + The folder path where pipelines are created. + + .EXAMPLE + Get-YamlPipelineDefinition -PipelineSourcePath './pipelines' -ProjectName P1 -RepositoryName R1 -BranchName main -PipelineTargetPath '/' + #> + [CmdletBinding()] + param + ( + [Parameter()] + [string]$PipelineSourcePath, + + [Parameter(Mandatory)] + [ValidateNotNullOrEmpty()] + [string]$ProjectName, + + [Parameter(Mandatory)] + [ValidateNotNullOrEmpty()] + [string]$RepositoryName, + + [Parameter(Mandatory)] + [ValidateNotNullOrEmpty()] + [string]$BranchName, + + [Parameter()] + [string]$PipelineTargetPath + ) + + $sourcePath = if ([string]::IsNullOrWhiteSpace($PipelineSourcePath)) { '.' } else { $PipelineSourcePath } + $resolvedSourcePath = (Resolve-Path -Path $sourcePath).Path + Write-Log "Identifying relevant Azure Pipelines under $resolvedSourcePath." + $ymlPipelines = Get-ChildItem -Path $resolvedSourcePath -Recurse -File -Filter 'pipeline.yml' | + Sort-Object FullName + Write-Log "Found $($ymlPipelines.Count) YAML Pipeline(s) in $resolvedSourcePath." + + $pipelinesArray = @() + foreach ($pipeline in $ymlPipelines) { + $ymlPath = [IO.Path]::GetRelativePath($resolvedSourcePath, $pipeline.FullName).Replace('\', '/') + $parentFolderName = Split-Path -Path (Split-Path -Path $pipeline.FullName -Parent) -Leaf + $pipelineName = $parentFolderName # used as the pipeline name + + $pipeObj = [PSCustomObject]@{ + ProjectName = $ProjectName + RepositoryName = $RepositoryName + BranchName = $BranchName + FolderPath = $PipelineTargetPath + ymlPath = $ymlPath + parentFolderName = $parentFolderName + pipelineName = $pipelineName + } + + $pipelinesArray += $pipeObj + } + + Write-Output $pipelinesArray +} + +function New-AzPipelineDefinition { + <# + .SYNOPSIS + Create a single Azure Pipeline and, optionally, its Build Validation. + + .DESCRIPTION + Creates one Azure Pipeline from a discovered definition object. When + requested, also creates a branch Build Validation policy scoped to the + pipeline's path. + + .PARAMETER Pipeline + The pipeline definition object produced by Get-YamlPipelineDefinition. + + .PARAMETER OrganizationUrl + The full Azure DevOps organization URL. + + .PARAMETER CreateBuildValidation + Also create a Pull Request Build Validation policy for the new pipeline. + + .EXAMPLE + New-AzPipelineDefinition -Pipeline $def -OrganizationUrl $url -CreateBuildValidation + #> + [CmdletBinding(SupportsShouldProcess)] + param + ( + [Parameter(Mandatory)] + [ValidateNotNull()] + [psobject]$Pipeline, + + [Parameter(Mandatory)] + [ValidateNotNullOrEmpty()] + [string]$OrganizationUrl, + + [Parameter()] + [switch]$CreateBuildValidation + ) + + if (-not $PSCmdlet.ShouldProcess($Pipeline.pipelineName, 'Create Azure Pipeline')) { + return + } + + Write-Log "Creating Azure pipeline $($Pipeline.pipelineName)." + $pipelineResult = az pipelines create --project "$($Pipeline.ProjectName)" ` + --organization "$OrganizationUrl" ` + --repository "$($Pipeline.RepositoryName)" ` + --repository-type tfsgit ` + --branch "$($Pipeline.BranchName)" ` + --folder-path "$($Pipeline.FolderPath)" ` + --name "$($Pipeline.pipelineName)" ` + --yml-path "$($Pipeline.ymlPath)" ` + --skip-run + $pipelineObject = $pipelineResult | ConvertFrom-Json + + if ($CreateBuildValidation) { + $pathFilter = $Pipeline.ymlPath -replace 'pipeline.yml', '*' + Write-Log "Configuring branch Build Validation for $($Pipeline.pipelineName)." + az repos policy build create ` + --blocking true ` + --branch "$($Pipeline.BranchName)" ` + --build-definition-id $pipelineObject.id ` + --display-name "Check $($Pipeline.pipelineName)" ` + --manual-queue-only true ` + --queue-on-source-update-only true ` + --valid-duration 1440 ` + --path-filter $pathFilter ` + --repository-id $pipelineObject.repository.id ` + --enabled true + } +} +#endregion + +#region Execution +Write-Log "Executing $($MyInvocation.MyCommand.Name)." + try { - Write-Verbose "----------------------------------" - Write-Verbose "Installing Azure CLI extension devops" - az config set extension.use_dynamic_install=yes_without_prompt # to allow installing extensions without prompt - az extension add --upgrade -n azure-devops - - Write-Verbose "----------------------------------" - Write-Verbose "Check for availability of Azure CLI and the CLI extension for Azure DevOps" - $az = az - $az = az devops -h - - Write-Verbose "----------------------------------" - Write-Verbose "Trying to login to Azure DevOps project $OrganizationName/$ProjectName with a PAT" - $orgUrl = "https://dev.azure.com/$OrganizationName/" - $env:AZURE_DEVOPS_EXT_PAT = $PAT - Write-Output $env:AZURE_DEVOPS_EXT_PAT | az devops login - - Write-Verbose "----------------------------------" - Write-Verbose "Set default Azure DevOps configuration to $OrganizationName and $ProjectName" - az devops configure --defaults organization="$orgUrl" project="$ProjectName" --use-git-aliases true - - Write-Verbose "----------------------------------" - Write-Verbose "Get and list all Azure Pipelines in $PipelineTargetPath" - $azurePipelines = az pipelines list --organization $orgUrl --project $ProjectName --folder-path $PipelineTargetPath | ConvertFrom-Json | Sort-Object name - Write-Verbose "Found $($azurePipelines.Count) Azure Pipeline(s) in $ProjectName" - - Write-Verbose "----------------------------------" - Write-Verbose "Identify relevant Azure Pipelines to be updated" - $PipelineSourcePath = Join-Path (Get-Location).Path $PipelineSourcePath - $ymlPipelines = Get-ChildItem -Path $PipelineSourcePath -Recurse | Where-Object { $_.Name -like "pipeline.yml" } | Sort-Object FullName - Write-Verbose "Found $($ymlPipelines.Count) YAML Pipeline(s) in $PipelineSourcePath" - - $pipelinesArray = @() - foreach ($pipeline in $ymlPipelines) { - $pipeObj = New-Object -TypeName PSCustomObject - $fullYmlPath = $pipeline.fullname.replace("\", "/") - $pathSplit = $fullYmlPath.Split("/") - $ymlPath = $pathSplit[-5] + "/" + $pathSplit[-4] + "/" + $pathSplit[-3] + "/" + $pathSplit[-2] + "/" + $pathSplit[-1] # - $parentFolderName = $pathSplit[-3] # here we have the parent folder name - $pipelineName = $pathSplit[-3] # which we take for the pipeline name - $pipeObj | Add-Member -MemberType NoteProperty -Name ProjectName -Value $ProjectName - $pipeObj | Add-Member -MemberType NoteProperty -Name RepositoryName -Value $RepositoryName - $pipeObj | Add-Member -MemberType NoteProperty -Name BranchName -Value $BranchName - $pipeObj | Add-Member -MemberType NoteProperty -Name FolderPath -Value $PipelineTargetPath - $pipeObj | Add-Member -MemberType NoteProperty -Name ymlPath -Value $ymlPath - $pipeObj | Add-Member -MemberType NoteProperty -Name parentFolderName -Value $parentFolderName - $pipeObj | Add-Member -MemberType NoteProperty -Name pipelineName -Value $pipelineName - - $pipelinesArray += $pipeObj - } - - $pipelinesToBeSkipped = $pipelinesArray | Where-Object { $_.pipelineName -in $azurePipelines.name } - $pipelinesToBeUpdated = $pipelinesArray | Where-Object { $_.pipelineName -notin $azurePipelines.name } - - if ($pipelinesToBeUpdated.Count -gt 0) { - Write-Verbose "----------------------------------" - Write-Verbose "$($pipelinesToBeUpdated.Count) Pipeline(s) have been identified to be updated" - Write-Verbose "$($pipelinesToBeSkipped.Count) Pipeline(s) will be skipped" - } - else { - Write-Verbose "----------------------------------" - Write-Verbose "No Pipelines have been identified. Exiting." - exit - } - - foreach ($pipeline in $pipelinesToBeUpdated) { - Write-Verbose "----------------------------------" - Write-Verbose "Create Azure pipeline $($pipeline.pipelineName) ... " - $pipelineresult = az pipelines create --project "$($pipeline.ProjectName)" ` - --organization "$orgUrl" ` - --repository "$($pipeline.RepositoryName)" ` - --repository-type tfsgit ` - --branch "$($pipeline.BranchName)" ` - --folder-path "$($pipeline.FolderPath)" ` - --name "$($pipeline.pipelineName)" ` - --yml-path "$($pipeline.ymlPath)" ` - --skip-run - $pipelineobject = $pipelineresult | ConvertFrom-Json - if ($createBuildValidation) { - $pathFilter = $pipeline.ymlpath -replace 'pipeline.yml', '*' - Write-Verbose "----------------------------------" - Write-Verbose "Configuring Master branch Build Validation for $($pipeline.pipelineName)" - $buildvalidation = az repos policy build create ` - --blocking true ` - --branch master ` - --build-definition-id $pipelineobject.id ` - --display-name "Check $($pipeline.pipelineName)" ` - --manual-queue-only true ` - --queue-on-source-update-only true ` - --valid-duration 1440 ` - --path-filter $pathFilter ` - --repository-id $pipelineobject.repository.id ` - --enabled true + $orgUrl = "https://dev.azure.com/$OrganizationName/" + + Connect-AzDevOpsCli -OrganizationUrl $orgUrl -ProjectName $ProjectName -PAT $PAT + + $azurePipelines = Get-ExistingAzPipeline -OrganizationUrl $orgUrl -ProjectName $ProjectName -PipelineTargetPath $PipelineTargetPath + + $pipelinesArray = Get-YamlPipelineDefinition ` + -PipelineSourcePath $PipelineSourcePath ` + -ProjectName $ProjectName ` + -RepositoryName $RepositoryName ` + -BranchName $BranchName ` + -PipelineTargetPath $PipelineTargetPath + + $pipelinesToBeSkipped = $pipelinesArray | Where-Object { $_.pipelineName -in $azurePipelines.name } + $pipelinesToBeUpdated = $pipelinesArray | Where-Object { $_.pipelineName -notin $azurePipelines.name } + + if ($pipelinesToBeUpdated.Count -eq 0) { + Write-Log 'No Pipelines have been identified. Exiting.' + return + } + + Write-Log "$($pipelinesToBeUpdated.Count) Pipeline(s) have been identified to be updated." + Write-Log "$($pipelinesToBeSkipped.Count) Pipeline(s) will be skipped." + + foreach ($pipeline in $pipelinesToBeUpdated) { + New-AzPipelineDefinition -Pipeline $pipeline -OrganizationUrl $orgUrl -CreateBuildValidation:$CreateBuildValidation } - } - - Write-Verbose "----------------------------------" - Write-Verbose "$($pipelinesToBeUpdated.Count) Azure pipeline(s) created!" - if ($createBuildValidation) { - Write-Verbose "$($pipelinesToBeUpdated.Count) Pull Request Build Validation(s) created!" - } - Write-Verbose "$($pipelinesToBeSkipped.Count) Azure pipeline(s) skipped!" - $url = $orgUrl + $ProjectName + "/" + "_build?definitionScope=%5C$PipelineTargetPath" - Write-Verbose "----------------------------------" - Write-Verbose "Please check your Azure Pipelines here: $url..." + + Write-Log "$($pipelinesToBeUpdated.Count) Azure pipeline(s) created!" + if ($CreateBuildValidation) { + Write-Log "$($pipelinesToBeUpdated.Count) Pull Request Build Validation(s) created!" + } + Write-Log "$($pipelinesToBeSkipped.Count) Azure pipeline(s) skipped!" + + $url = $orgUrl + $ProjectName + '/' + '_build?definitionScope=%5C' + $PipelineTargetPath + Write-Log "Please check your Azure Pipelines here: $url" } catch { - Write-Verbose "----------------------------------" - Write-Warning ("Reason: [{0}]" -f $_.Exception.Message) -} \ No newline at end of file + Write-Log -Message ('Reason: [{0}]' -f $_.Exception.Message) -ErrorRecord $_ + throw +} + +Write-Log "Finished $($MyInvocation.MyCommand.Name)." +#endregion diff --git a/PowerShell/RepoTools.psm1 b/PowerShell/RepoTools.psm1 new file mode 100644 index 0000000..64ae55d --- /dev/null +++ b/PowerShell/RepoTools.psm1 @@ -0,0 +1,292 @@ +#Requires -Version 7.0 + +<# +.SYNOPSIS + Clone-or-update helpers for GitHub and Azure DevOps repositories. + +.DESCRIPTION + Provides advanced functions that mirror a remote organisation's repositories + into a local folder: missing repositories are cloned, existing ones are + switched to their default branch and fast-forwarded. Both providers share a + single private helper (Update-GitRepository) so the git clone/checkout/pull + behaviour is identical regardless of source. + + Import it from a script with: + + Import-Module "$PSScriptRoot/RepoTools.psm1" -Force + + Requires the git CLI on PATH. Update-GitHubRepos optionally uses the gh CLI + to enumerate repositories; Update-AdoRepos calls the Azure DevOps REST API + with a personal access token. + +.NOTES + Author: Sebastian Gräf + Repo: https://github.com/segraef/Scripts +#> + +Import-Module "$PSScriptRoot/Write-Log.psm1" -Force + +function Update-GitRepository { + <# + .SYNOPSIS + Clone a repository if missing, otherwise checkout its default branch and pull. + + .DESCRIPTION + Shared worker for the provider-specific functions. If the target folder is + absent or empty the repository is cloned from RepositoryUrl. If it already + contains a working tree the default branch is checked out and the latest + changes are pulled. State-changing git operations honour -WhatIf/-Confirm. + + .PARAMETER RepositoryUrl + The clone URL of the repository (HTTPS or SSH). + + .PARAMETER RepositoryPath + The local folder the repository is (or will be) cloned into. + + .PARAMETER DefaultBranch + The branch to checkout before pulling on an existing clone. Defaults to 'main'. + + .PARAMETER Name + A friendly repository name used in log messages and ShouldProcess prompts. + + .EXAMPLE + Update-GitRepository -RepositoryUrl 'https://github.com/Azure/foo.git' -RepositoryPath './Azure/foo' -Name 'foo' + Clones foo if missing, otherwise checks out main and pulls. + + .NOTES + Author: Sebastian Gräf + Repo: https://github.com/segraef/Scripts + #> + [CmdletBinding(SupportsShouldProcess)] + param + ( + [Parameter(Mandatory)] + [ValidateNotNullOrEmpty()] + [string]$RepositoryUrl, + + [Parameter(Mandatory)] + [ValidateNotNullOrEmpty()] + [string]$RepositoryPath, + + [Parameter()] + [ValidateNotNullOrEmpty()] + [string]$DefaultBranch = 'main', + + [Parameter()] + [ValidateNotNullOrEmpty()] + [string]$Name = (Split-Path -Path $RepositoryPath -Leaf) + ) + + $isPresent = Test-Path -Path $RepositoryPath + $isEmpty = $isPresent -and -not (Get-ChildItem -Path $RepositoryPath -Force | Select-Object -First 1) + + if (-not $isPresent -or $isEmpty) { + if ($PSCmdlet.ShouldProcess($RepositoryPath, "Clone repository '$Name'")) { + Write-Log "Cloning '$Name' into '$RepositoryPath'." + try { + New-Item -ItemType Directory -Path $RepositoryPath -Force | Out-Null + git clone $RepositoryUrl $RepositoryPath + if ($LASTEXITCODE -ne 0) { + throw "git clone exited with code $LASTEXITCODE for '$Name'." + } + } + catch { + Write-Log -Message "Failed to clone '$Name'." -ErrorRecord $_ + throw + } + } + return + } + + if ($PSCmdlet.ShouldProcess($RepositoryPath, "Checkout '$DefaultBranch' and pull '$Name'")) { + Write-Log "Updating '$Name' (checkout '$DefaultBranch', pull)." + try { + git -C $RepositoryPath checkout $DefaultBranch + if ($LASTEXITCODE -ne 0) { + throw "git checkout '$DefaultBranch' exited with code $LASTEXITCODE for '$Name'." + } + git -C $RepositoryPath pull + if ($LASTEXITCODE -ne 0) { + throw "git pull exited with code $LASTEXITCODE for '$Name'." + } + } + catch { + Write-Log -Message "Failed to update '$Name'." -ErrorRecord $_ + throw + } + } +} + +function Update-GitHubRepos { + <# + .SYNOPSIS + Clone or update a set of GitHub repositories for an organisation. + + .DESCRIPTION + Mirrors the named GitHub repositories into a local folder under + //. Missing repositories are cloned over + HTTPS, existing ones are checked out to the default branch and pulled via + the shared Update-GitRepository helper. + + .PARAMETER TargetFolder + The root folder the repositories are cloned or updated under. + + .PARAMETER Organization + The GitHub organisation (or user) the repositories belong to. + + .PARAMETER Repos + The repository names to clone or update. + + .PARAMETER DefaultBranch + The branch to checkout before pulling on existing clones. Defaults to 'main'. + + .EXAMPLE + Update-GitHubRepos -TargetFolder './Git' -Organization 'Azure' -Repos @('bicep','azure-cli') + Clones or updates the two named repositories under ./Git/Azure. + + .EXAMPLE + $repos = gh repo list azure -L 5000 --json name --jq '.[].name' | Select-String -Pattern 'terraform-azurerm-avm' + Update-GitHubRepos -TargetFolder './Git' -Organization 'Azure' -Repos $repos + Pipes a filtered repository list from the gh CLI into the updater. + + .NOTES + Author: Sebastian Gräf + Repo: https://github.com/segraef/Scripts + #> + [CmdletBinding(SupportsShouldProcess)] + [Diagnostics.CodeAnalysis.SuppressMessageAttribute( + 'PSUseSingularNouns', '', + Justification = 'Plural noun is the established, caller-facing command name.')] + param + ( + [Parameter(Mandatory)] + [ValidateNotNullOrEmpty()] + [string]$TargetFolder, + + [Parameter(Mandatory)] + [ValidateNotNullOrEmpty()] + [string]$Organization, + + [Parameter(Mandatory)] + [ValidateNotNullOrEmpty()] + [string[]]$Repos, + + [Parameter()] + [ValidateNotNullOrEmpty()] + [string]$DefaultBranch = 'main' + ) + + Write-Log "Found $($Repos.Count) GitHub repositories for '$Organization'." + + foreach ($repo in $Repos) { + $repoName = "$repo".Trim() + $repoPath = Join-Path -Path $TargetFolder -ChildPath (Join-Path -Path $Organization -ChildPath $repoName) + $repoUrl = "https://github.com/$Organization/$repoName.git" + + Update-GitRepository -RepositoryUrl $repoUrl -RepositoryPath $repoPath -DefaultBranch $DefaultBranch -Name $repoName + } +} + +function Update-AdoRepos { + <# + .SYNOPSIS + Clone or update every repository across all projects in an Azure DevOps organisation. + + .DESCRIPTION + Enumerates the projects in an Azure DevOps organisation via the REST API, + then clones or updates each project's repositories into + //. Missing repositories are cloned, existing + ones are checked out to the default branch and pulled via the shared + Update-GitRepository helper. + + .PARAMETER Organization + The Azure DevOps organisation name (the segment after dev.azure.com/). + + .PARAMETER TargetFolder + The root folder the repositories are cloned or updated under. + + .PARAMETER Pat + The personal access token used to authenticate against the REST API. + + .PARAMETER DefaultBranch + The branch to checkout before pulling on existing clones. Defaults to 'main'. + + .EXAMPLE + $pat = Read-Host -AsSecureString 'PAT' + Update-AdoRepos -Organization 'contoso' -TargetFolder 'C:/Repos' -Pat $pat + Clones or updates every repository in every project of the contoso organisation. + + .NOTES + Author: Sebastian Gräf + Repo: https://github.com/segraef/Scripts + #> + [CmdletBinding(SupportsShouldProcess)] + [Diagnostics.CodeAnalysis.SuppressMessageAttribute( + 'PSUseSingularNouns', '', + Justification = 'Plural noun is the established, caller-facing command name.')] + param + ( + [Parameter(Mandatory)] + [ValidateNotNullOrEmpty()] + [string]$Organization, + + [Parameter(Mandatory)] + [ValidateNotNullOrEmpty()] + [string]$TargetFolder, + + [Parameter(Mandatory)] + [securestring]$Pat, + + [Parameter()] + [ValidateNotNullOrEmpty()] + [string]$DefaultBranch = 'main' + ) + + $plainPat = [System.Net.NetworkCredential]::new('', $Pat).Password + $authHeader = @{ + Authorization = 'Basic ' + [Convert]::ToBase64String([Text.Encoding]::ASCII.GetBytes(":$plainPat")) + } + + Write-Log "Getting projects for organisation '$Organization'." + try { + $projectsUri = "https://dev.azure.com/$Organization/_apis/projects?api-version=6.0" + $projects = (Invoke-RestMethod -Uri $projectsUri -Headers $authHeader).value + } + catch { + Write-Log -Message "Failed to list projects for '$Organization'." -ErrorRecord $_ + throw + } + + Write-Log "Found $($projects.Count) projects: $($projects.name -join ', ')" + + foreach ($project in $projects) { + $projectFolder = Join-Path -Path $TargetFolder -ChildPath $project.name + + if (-not (Test-Path -Path $projectFolder)) { + if ($PSCmdlet.ShouldProcess($projectFolder, 'Create project folder')) { + Write-Log "Creating folder '$projectFolder'." + New-Item -ItemType Directory -Path $projectFolder -Force | Out-Null + } + } + + Write-Log "Getting repos for project '$($project.name)'." + try { + $reposUri = "https://dev.azure.com/$Organization/$($project.name)/_apis/git/repositories?api-version=6.0" + $reposUri = $reposUri -replace ' ', '%20' + $repos = (Invoke-RestMethod -Uri $reposUri -Headers $authHeader).value + } + catch { + Write-Log -Message "Failed to list repos for project '$($project.name)'." -ErrorRecord $_ + throw + } + + Write-Log "Found $($repos.Count) repos: $($repos.name -join ', ')" + + foreach ($repo in $repos) { + $repoPath = Join-Path -Path $projectFolder -ChildPath $repo.name + Update-GitRepository -RepositoryUrl $repo.remoteUrl -RepositoryPath $repoPath -DefaultBranch $DefaultBranch -Name $repo.name + } + } +} + +Export-ModuleMember -Function Update-GitHubRepos, Update-AdoRepos diff --git a/PowerShell/Set-AzPolicyDefinitions.ps1 b/PowerShell/Set-AzPolicyDefinitions.ps1 index 25cc27a..bcfcfdd 100644 --- a/PowerShell/Set-AzPolicyDefinitions.ps1 +++ b/PowerShell/Set-AzPolicyDefinitions.ps1 @@ -1,100 +1,148 @@ -#Requires -Version 5.1 +#Requires -Version 7.0 <# .SYNOPSIS - + Create Azure Policy definitions from a folder of policy JSON files. .DESCRIPTION - + Reads every JSON file in the supplied folder and registers each one as an + Azure Policy definition under the specified management group. The file base + name becomes both the policy name and display name; the JSON body supplies + the policy rule, parameters and resource mode. -.PARAMETER - + Prerequisites: the Az PowerShell module must be installed and an authenticated + Azure session (Connect-AzAccount) must be active with rights to create policy + definitions at the chosen management group scope. + +.PARAMETER PolicyFolder + Path to the folder containing the policy definition JSON files. Each file is + processed in turn. + +.PARAMETER ManagementGroupName + Name (ID) of the management group at which the policy definitions are created. + +.PARAMETER PolicyDescription + Description applied to every policy definition created in this run. .INPUTS - + None. This script does not accept pipeline input. .OUTPUTS - .log> + Microsoft.Azure.Commands.ResourceManager.Cmdlets.Implementation.Policy.PsPolicyDefinition + One object per policy definition created. -.NOTES - Version: 1.0 - Author: - Creation Date: - Purpose/Change: Initial script development +.EXAMPLE + ./Set-AzPolicyDefinitions.ps1 -PolicyFolder './policies' -ManagementGroupName 'mg-corp' + Creates a policy definition for every JSON file in ./policies under the + 'mg-corp' management group, using the default description. .EXAMPLE - + ./Set-AzPolicyDefinitions.ps1 -PolicyFolder './policies' -ManagementGroupName 'mg-corp' -PolicyDescription 'Apply Diagnostics Settings' + Creates the policy definitions with a custom description. + +.NOTES + Author: Sebastian Gräf + Repo: https://github.com/segraef/Scripts + Version history is tracked in git, not in this header. #> #region Parameters - -[CmdletBinding()] +[CmdletBinding(SupportsShouldProcess)] param ( - [Parameter()] - [String]$policyFolder = ".\", - [Parameter()] - [String]$managementGroup = (Get-AzManagementGroup | Out-GridView -PassThru), - [Parameter()] - [String]$policyDescription = "Apply Diagnostics Settings" -) - -#endregion + [Parameter(Mandatory)] + [ValidateNotNullOrEmpty()] + [string]$PolicyFolder, -#region Initialisations - -$ErrorActionPreference = "Continue" -$VerbosePreference = "Continue" - -# Dot Source required Function Libraries -Import-Module ..\Write-Log.ps1 + [Parameter(Mandatory)] + [ValidateNotNullOrEmpty()] + [string]$ManagementGroupName, + [Parameter()] + [ValidateNotNullOrEmpty()] + [string]$PolicyDescription = 'Apply Diagnostics Settings' +) #endregion -#region Declarations +#region Initialisation +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +Import-Module "$PSScriptRoot/Write-Log.psm1" -Force #endregion #region Functions +function New-PolicyDefinition { + <# + .SYNOPSIS + Create a single Azure Policy definition from a policy JSON file. + + .DESCRIPTION + Parses the supplied JSON file, extracts the policy rule, parameters and + mode, then creates an Azure Policy definition named after the file base + name at the given management group scope. + + .PARAMETER Path + Full path to the policy definition JSON file. + + .PARAMETER ManagementGroupName + Name (ID) of the management group at which the definition is created. + + .PARAMETER Description + Description applied to the policy definition. + + .EXAMPLE + New-PolicyDefinition -Path './policies/audit-vm.json' -ManagementGroupName 'mg-corp' -Description 'Apply Diagnostics Settings' + Creates the 'audit-vm' policy definition under the 'mg-corp' management group. + #> + [CmdletBinding(SupportsShouldProcess)] + [OutputType([object])] + param + ( + [Parameter(Mandatory)] + [ValidateNotNullOrEmpty()] + [string]$Path, + + [Parameter(Mandatory)] + [ValidateNotNullOrEmpty()] + [string]$ManagementGroupName, + + [Parameter(Mandatory)] + [ValidateNotNullOrEmpty()] + [string]$Description + ) + + $name = [System.IO.Path]::GetFileNameWithoutExtension($Path) + Write-Log "Processing policy definition '$name' from '$Path'." -function FunctionName { - Param() - - begin { - Write-Log "Let's start !" - } - - process { try { - Write-Output "Hello Template !" + $json = Get-Content -Path $Path -Raw | ConvertFrom-Json + $policyRule = $json.policyRule | ConvertTo-Json -Depth 8 | ForEach-Object { [System.Text.RegularExpressions.Regex]::Unescape($_) } + $parameters = $json.parameters | ConvertTo-Json -Depth 8 | ForEach-Object { [System.Text.RegularExpressions.Regex]::Unescape($_) } } - catch { - Write-Output $_ - Write-Log $_ -Warning + Write-Log -Message "Failed to parse policy file '$Path'." -ErrorRecord $_ + throw } - } - end { - if ($?) { - Write-Log "Completed successfully !" + if ($PSCmdlet.ShouldProcess($name, 'Create Azure policy definition')) { + try { + New-AzPolicyDefinition -Name $name -DisplayName $name -Policy $policyRule -Description $Description -Parameter $parameters -Mode $json.mode -ManagementGroupName $ManagementGroupName + } + catch { + Write-Log -Message "Failed to create policy definition '$name'." -ErrorRecord $_ + throw + } } - } } - #endregion #region Execution +Write-Log "Executing $($MyInvocation.MyCommand.Name)." -Write-Log "Executing $($MyInvocation.MyCommand.Name)" - -foreach ($item in (Get-Childitem $policyFolder)) { - $json = Get-Content $item.FullName | ConvertFrom-Json - # $mode = $json.mode | ConvertTo-Json - $policyRule = $json.policyRule | ConvertTo-Json -Depth 8 | ForEach-Object { [System.Text.RegularExpressions.Regex]::Unescape($_) } - $parameters = $json.parameters | ConvertTo-Json -Depth 8 | ForEach-Object { [System.Text.RegularExpressions.Regex]::Unescape($_) } - New-AzPolicyDefinition -Name $item.BaseName -DisplayName $item.BaseName -Policy $policyRule -Description $policyDescription -Parameter $parameters -Mode $json.mode -ManagementGroupName $managementGroup.Name +foreach ($item in (Get-ChildItem -Path $PolicyFolder -File)) { + New-PolicyDefinition -Path $item.FullName -ManagementGroupName $ManagementGroupName -Description $PolicyDescription } -Write-Log "Finished executing $($MyInvocation.MyCommand.Name)" - +Write-Log "Finished executing $($MyInvocation.MyCommand.Name)." #endregion diff --git a/PowerShell/Snippets/ADO-BuiltInVariables.ps1 b/PowerShell/Snippets/ADO-BuiltInVariables.ps1 deleted file mode 100644 index e98348d..0000000 --- a/PowerShell/Snippets/ADO-BuiltInVariables.ps1 +++ /dev/null @@ -1,13 +0,0 @@ -# this is inline code -env | sort - -# code trimmed for brevity from azure-pipelines.yml - - steps: # 'Steps' section is to be used inside 'job' section. - – task: Bash@3 - inputs: - targetType: 'inline' - script: 'env | sort' - - -https://docs.microsoft.com/en-us/azure/devops/pipelines/build/variables?view=azure-devops&tabs=yaml \ No newline at end of file diff --git a/PowerShell/Snippets/AVM-ModuleTester.ps1 b/PowerShell/Snippets/AVM-ModuleTester.ps1 deleted file mode 100644 index 8428b52..0000000 --- a/PowerShell/Snippets/AVM-ModuleTester.ps1 +++ /dev/null @@ -1,81 +0,0 @@ -# AVM Module Tester Script -# Before running this script, make sure to: -# 1. Replace '' with your actual Azure subscription ID -# 2. Replace '' with your desired naming prefix -# 3. Replace '' with your Azure AD tenant ID -# 4. Ensure you have the required Azure PowerShell modules installed - -# Start pwsh if not started yet - -# pwsh - -# Set default directory -$folder = "Git/GitHub/Azure/bicep-registry-modules" # location of your local clone of bicep-registry-modules - -# Ensure Azure PowerShell authentication -if (-not (Get-AzContext)) { - Write-Output "No Azure context found. Please authenticate..." - Connect-AzAccount -} - -# Set the subscription context (update with your actual subscription ID) -$subscriptionId = '' # Replace with your actual subscription ID -if ($subscriptionId -ne '') { - Set-AzContext -SubscriptionId $subscriptionId -} - -# Dot source functions -. $folder/utilities/tools/Set-AVMModule.ps1 -. $folder/utilities/tools/Test-ModuleLocally.ps1 - -# Variables - -$modules = @( - "web/site" # 5599 - # "communication/communication-service" # 5598 -) - -# Generate Readme - -foreach ($module in $modules) { - Write-Output "Generating ReadMe for module $module" - Set-AVMModule -ModuleFolderPath "$folder/avm/res/$module" -Recurse - - # Set up test settings - - $testcases = "functionApp.defaults", "webApp.max" #, "waf-aligned", "max", "defaults" - # $testcase = "all" - - $TestModuleLocallyInput = @{ - TemplateFilePath = "$folder/avm/res/$module/main.bicep" - PesterTest = $true - ValidationTest = $true - DeploymentTest = $true - ValidateOrDeployParameters = @{ - Location = 'australiaeast' - SubscriptionId = $subscriptionId - RemoveDeployment = $true - } - AdditionalTokens = @{ - namePrefix = 'asf3re' # Replace with your prefix - TenantId = '' # Replace with your tenant ID - } - } - - # Run tests - # if testcase is 'all' browse all folders in tests/e2e - if ($testcase -eq "all") { - $testcases = Get-ChildItem -Path "$folder/avm/res/$module/tests/e2e" -Directory | ForEach-Object { $_.Name } - } - foreach ($testcase in $testcases) { - Write-Output "Running test case $testcase on module $module" - $TestModuleLocallyInput.ModuleTestFilePath = "$folder/avm/res/$module/tests/e2e/$testcase/main.test.bicep" - - try { - Test-ModuleLocally @TestModuleLocallyInput - } - catch { - Write-Output $_.Exception | Format-List -Force - } - } -} diff --git a/PowerShell/Snippets/Add-GitHubIssue.ps1 b/PowerShell/Snippets/Add-GitHubIssue.ps1 deleted file mode 100644 index 837491a..0000000 --- a/PowerShell/Snippets/Add-GitHubIssue.ps1 +++ /dev/null @@ -1,31 +0,0 @@ -$tfrepos = gh repo list azure -L 5000 --json name --jq '.[].name' | findstr terraform-azurerm-avm - -$allItems = @() -Write-Host "Fetching issues and PRs for $($tfrepos.Count) repos" -foreach ($repo in $tfrepos) -{ - Write-Host "Fetching issues and PRs for $repo" - - # Fetch issues - $issues = gh issue list -R "Azure/$repo" --json number,title,url | ConvertFrom-Json - foreach ($issue in $issues) { - $issue | Add-Member -NotePropertyName "repo" -NotePropertyValue $repo - $issue | Add-Member -NotePropertyName "type" -NotePropertyValue "issue" - } - $allItems += $issues -} - -$allItems | ConvertTo-Json - -$allItems | Where-Object { $_.type -eq "issue" } | Select-Object title,url - -# Add all issues to project' - -foreach ($issue in $allItems) -{ - if($issue.repo -ne "terraform-azurerm-avm-template") - { - Write-Host "Adding Issue $($issue.title) on repo $($issue.repo)" - gh issue edit $issue.url --add-project "AVM - Module Issues" - } -} diff --git a/PowerShell/Snippets/Add-PullRequests.ps1 b/PowerShell/Snippets/Add-PullRequests.ps1 deleted file mode 100644 index 17abe5f..0000000 --- a/PowerShell/Snippets/Add-PullRequests.ps1 +++ /dev/null @@ -1,26 +0,0 @@ -function Add-PullRequests { - - [CmdletBinding()] - Param( - [Parameter()] - [string]$org = "x00", - [Parameter()] - [string]$project = "Modules", - [Parameter()] - [string]$token = "token", - [Parameter()] - [string]$branchName = "users/segraef/provider-upgrade" - ) - - az extension add --name azure-devops - - Write-Output $token | az devops login --organization https://dev.azure.com/$org - az devops configure --defaults organization=https://dev.azure.com/$org project=$project - $repos = az repos list | ConvertFrom-Json - - # Create Pull Request on all repos - foreach ($repo in $repos) { - Write-Output $repo.name - az repos pr create --repository $repo.name --source-branch $branchName --open --output table - } -} diff --git a/PowerShell/Snippets/Cleanup-LocalGitBranches.ps1 b/PowerShell/Snippets/Cleanup-LocalGitBranches.ps1 deleted file mode 100644 index 2869819..0000000 --- a/PowerShell/Snippets/Cleanup-LocalGitBranches.ps1 +++ /dev/null @@ -1,10 +0,0 @@ -$folder = Get-Location # '.' -$localRepos = Get-ChildItem $folder.Path -Directory -recurse | select * - -foreach($repo in $localRepos) { - "$($repo.FullName)" - Set-Location "$($repo.FullName)" - git tag -d 1.0.0 - git checkout main; git pull; git remote update origin --prune; git branch -vv | Select-String -Pattern ": gone]" | % { $_.toString().Trim().Split(" ")[0]} | % { git branch -D $_ } -} -Set-Location $folder.Path diff --git a/PowerShell/Snippets/Clone-Repos.ps1 b/PowerShell/Snippets/Clone-Repos.ps1 deleted file mode 100644 index ff16701..0000000 --- a/PowerShell/Snippets/Clone-Repos.ps1 +++ /dev/null @@ -1,37 +0,0 @@ -Param( - [string]$destinationFolder = ".", - [string]$org = "123", - [array]$projects = ( - "Modules") - - # Make sure you have the Azure CLI installed (az extension add --name azure-devops) and logged in via az login or az devops login - # If you face /_apis authentication issues make sure to login via az login --allow-no-subscriptions - - az devops configure --defaults organization=https://dev.azure.com/$org - # $projects = az devops project list --organization=https://dev.azure.com/$org | ConvertFrom-Json - - foreach ($project in $projects) { - $repos = az repos list --project $project | ConvertFrom-Json - foreach ($repo in $repos) { - Write-Output "Repository [$($repo.name)] in project [$($project)]" - If (!(test-path -PathType container $destinationFolder)) { - New-Item -ItemType Directory -Path $destinationFolder - } - git clone $($repo.remoteUrl) $destinationFolder/$($project)/$($repo.name) - } - } - - # clone repos in github - - $destinationFolder = "." - - $tfrepos = gh repo list azure -L 5000 --json name --jq '.[].name' | Select-String -Pattern "terraform-azurerm-avm" - $org = 'Azure' - - foreach ($repo in $tfrepos) { - If (!(test-path -PathType container $destinationFolder/$($org)/$($repo))) { - New-Item -ItemType Directory -Path $destinationFolder/$($org)/$($repo) - } - Write-Output "https://github.com/$org/$repo.git into $destinationFolder/$($org)/$($repo)" - git clone "https://github.com/$org/$repo.git" $destinationFolder/$($org)/$($repo) - } diff --git a/PowerShell/Snippets/Create-Branches.ps1 b/PowerShell/Snippets/Create-Branches.ps1 deleted file mode 100644 index ad69b78..0000000 --- a/PowerShell/Snippets/Create-Branches.ps1 +++ /dev/null @@ -1,16 +0,0 @@ -Param( - [string]$destinationFolder = "." -) - -$repos = Get-ChildItem -Path $destinationFolder -Directory - -foreach($repo in $repos) { - $($repo.FullName) - Set-Location "$($repo.FullName)"; - git checkout -b users/segraef/provider-upgrade; - git push --set-upstream origin users/segraef/provider-upgrade; - git add .; - git commit -m "provider upgrade"; - git push; -} - diff --git a/PowerShell/Snippets/Create-BuildValidationPolicies.ps1 b/PowerShell/Snippets/Create-BuildValidationPolicies.ps1 deleted file mode 100644 index 1fc977c..0000000 --- a/PowerShell/Snippets/Create-BuildValidationPolicies.ps1 +++ /dev/null @@ -1,115 +0,0 @@ -# Define the list of Azure DevOps project names -$organization = "org1" -$pat = "" -$projects = @( - [PSCustomObject]@{ - name = "project1" - } -) -$VerbosePreference = 'Continue' - -# Log in to Azure DevOps using PAT -$env:AZURE_DEVOPS_EXT_PAT = "$pat" - -# Function to get all projects in the organization -function Get-AdoProjects { - $uri = "https://dev.azure.com/$organization/_apis/projects?api-version=6.0" - $response = Invoke-RestMethod -Uri $uri -Headers @{Authorization = "Basic " + [Convert]::ToBase64String([Text.Encoding]::ASCII.GetBytes(":$pat")) } - return $response.value -} - -# Function to get repositories for a given project -function Get-AdoRepositories($project) { - $uri = "https://dev.azure.com/$organization/$project/_apis/git/repositories?api-version=6.0" - $uri = $uri -replace " ", "%20" - $response = Invoke-RestMethod -Uri $uri -Headers @{Authorization = "Basic " + [Convert]::ToBase64String([Text.Encoding]::ASCII.GetBytes(":$pat")) } - return $response.value -} - -# Function to get all Azure Pipelines -function Get-AdoPipelines($project) { - $uri = "https://dev.azure.com/$organization/$project/_apis/pipelines?api-version=6.0" - $uri = $uri -replace " ", "%20" - $response = Invoke-RestMethod -Uri $uri -Headers @{Authorization = "Basic " + [Convert]::ToBase64String([Text.Encoding]::ASCII.GetBytes(":$pat")) } - return $response.value -} - -# Function to get all build policies -function Get-AdoBuildPolicies($project) { - $uri = "https://dev.azure.com/$organization/$project/_apis/policy/configurations?api-version=6.0" - $uri = $uri -replace " ", "%20" - $response = Invoke-RestMethod -Uri $uri -Headers @{Authorization = "Basic " + [Convert]::ToBase64String([Text.Encoding]::ASCII.GetBytes(":$pat")) } - return $response.value -} - -function CreateBuildValidationPolicy($project, $repo, $buildDefinition) { - # check matching policies - $matchingPolicies = $buildPolicies | Where-Object { $_.settings.buildDefinitionId -eq $buildDefinition.id } - if ($matchingPolicies) { - Write-Verbose "Policy already exists for $($repo.name) using pipeline $($buildDefinition.name). Updating." - az repos policy build update ` - --id $matchingPolicies.id ` - --blocking $false ` - --branch main ` - --build-definition-id $buildDefinition.id ` - --display-name $repo.name ` - --enabled $true ` - --manual-queue-only $false ` - --queue-on-source-update-only $false ` - --repository-id $repo.id ` - --valid-duration 0 ` - --project $project.name - - # Build Validation Policy - # --blocking $false - Policy Requirement: Optional - # --manual-queue-only $false - Trigger: Manual - # --valid-duration 0 - Build expiration: Immediately when main is updated - } else { - Write-Verbose "Creating build validation policy for $($repo.name) using pipeline $($buildDefinition.name)." - az repos policy build create ` - --blocking $false ` - --branch main ` - --build-definition-id $buildDefinition.id ` - --display-name $repo.name ` - --enabled $true ` - --manual-queue-only $false ` - --queue-on-source-update-only $false ` - --repository-id $repo.id ` - --valid-duration 0 ` - --project $project.name - } -} - -# Main script -Write-Verbose "Getting projects ..." -if ($projects.Count -eq 0) { - Write-Verbose "Getting all projects ..." - $projects = Get-AdoProjects -} else { - Write-Verbose "Using provided project(s): $($projects.name)" -} -Write-Verbose "Found $($projects.Count) project(s): $($projects.name)" -foreach ($project in $projects) { - Write-Verbose "Getting repos for $($project.name) ..." - $repos = Get-AdoRepositories -project $project.name - Write-Verbose "Found $($repos.Count) repos." - $pipelines = Get-AdoPipelines -project $project.name - Write-Verbose "Found $($pipelines.Count) pipelines." - $buildPolicies = Get-AdoBuildPolicies -project $project.name - Write-Verbose "Found $($buildPolicies.Count) build policies." - # Check matching pipelines - $checkedRepos = $repos | Where-Object { $_.name -in $pipelines.name } - Write-Verbose "Found $($checkedRepos.Count) repos with matching pipelines." - - Read-Host "Press Enter to continue" - - foreach ($repo in $repos) { - $buildDefinition = $pipelines | Where-Object { $_.name -eq "$($repo.name)" } - # $response = Read-Host "Do you want to create build validation for repo $($repo.name) using pipeline $($buildDefinition.name)? (y/n)" - # if ($response -eq 'y') { - CreateBuildValidationPolicy -repo $repo -project $project -buildDefinition $buildDefinition - # } else { - # Write-Verbose "Skipping $($repo.name)" - # } - } -} diff --git a/PowerShell/Snippets/Create-SP.ps1 b/PowerShell/Snippets/Create-SP.ps1 deleted file mode 100644 index dba6b72..0000000 --- a/PowerShell/Snippets/Create-SP.ps1 +++ /dev/null @@ -1,16 +0,0 @@ -Param( - [string]$spRole = "Contributor", - [string]$spScope = "/subscriptions/$subscriptionId" -) - -# PowerShell -$spName = (Get-Random -SetSeed 1234 -Minimum 100000 -Maximum 999999).ToString() + "-sp" -$sp = New-AzADServicePrincipal -DisplayName $spName -New-AzRoleAssignment -RoleDefinitionName $spRole -ServicePrincipalName $sp.ApplicationId -Scope $spScope - -# Azure CLI -spName=$(shuf -i 100000-999999 -n 1)-sp -sp=$(az ad sp create-for-rbac --name $spName --role $spRole --scopes $spScope) -echo $sp | jq -r .appId -echo $sp | jq -r .password -echo $sp | jq -r .tenant diff --git a/PowerShell/Snippets/Delete-Items.ps1 b/PowerShell/Snippets/Delete-Items.ps1 deleted file mode 100644 index a2c3758..0000000 --- a/PowerShell/Snippets/Delete-Items.ps1 +++ /dev/null @@ -1,6 +0,0 @@ -# With -Directory for folders and without for files -$WhatIfPreference = 'False' - -# $searchprefix = '.git' -$searchprefix = '.pre*' -Get-ChildItem -Path $searchprefix -Recurse -Hidden | ForEach-Object -Process { Remove-Item -Path $_.FullName -Verbose -Recurse -Force} diff --git a/PowerShell/Snippets/Generate-Text.ps1 b/PowerShell/Snippets/Generate-Text.ps1 deleted file mode 100644 index 1bf7032..0000000 --- a/PowerShell/Snippets/Generate-Text.ps1 +++ /dev/null @@ -1,6 +0,0 @@ -$names = Get-ChildItem -foreach($name in $names) { - $x = $name.BaseName -replace "(?-i)[A-Z]",'-$&' - $x = $x.ToLower() - Write-output """terraform-azurerm$x""," -} diff --git a/PowerShell/Snippets/Get-AllActions.ps1 b/PowerShell/Snippets/Get-AllActions.ps1 deleted file mode 100644 index 6f882c6..0000000 --- a/PowerShell/Snippets/Get-AllActions.ps1 +++ /dev/null @@ -1,2 +0,0 @@ -$actions = Get-AzProviderOperation -OperationSearchString '*' -$actions | Where-Object {$_.Operation -like '*read*'} | Select-Object Operation | Export-Csv -Path 'Get-AllActions.csv' -NoTypeInformation -Force diff --git a/PowerShell/Snippets/Get-RBACDetails.ps1 b/PowerShell/Snippets/Get-RBACDetails.ps1 deleted file mode 100644 index 124eba2..0000000 --- a/PowerShell/Snippets/Get-RBACDetails.ps1 +++ /dev/null @@ -1,44 +0,0 @@ -# Login to Azure -Connect-AzAccount - -<# -Function to fetch RBAC details recursively from Management Group to Resource Groups and export to CSV -0. Create empty array to store RBAC details -1. Fetch Management Group info and RBAC assignments -2. Fetch Subscriptions and RBAC assignments under the Management Group -3. Fetch Resource Groups and RBAC assignments under the Subscription - - -#> - -Function Get-RBACDetails { - [CmdletBinding()] - param ( - [Parameter(Mandatory=$true)] - [string]$ManagementGroup - ) - $RBACDetails = @() - $MGInfo = Get-AzManagementGroup -GroupName $ManagementGroup - $MGRBAC = Get-AzManagementGroupRoleAssignment -GroupId $ManagementGroup - $MGInfo | Add-Member -MemberType NoteProperty -Name "Type" -Value "Management Group" -Force - $MGInfo | Add-Member -MemberType NoteProperty -Name "RoleAssignment" -Value $MGRBAC -Force - $RBACDetails += $MGInfo - $Subscriptions = Get-AzManagementGroupSubscriptions -GroupId $ManagementGroup - foreach ($Subscription in $Subscriptions) { - $SubRBAC = Get-AzRoleAssignment -Scope $Subscription.Id - $Subscription | Add-Member -MemberType NoteProperty -Name "Type" -Value "Subscription" -Force - $Subscription | Add-Member -MemberType NoteProperty -Name "RoleAssignment" -Value $SubRBAC -Force - $RBACDetails += $Subscription - $ResourceGroups = Get-AzResourceGroup -SubscriptionId $Subscription.Id - foreach ($ResourceGroup in $ResourceGroups) { - $RGInfo = Get-AzResourceGroup -Name $ResourceGroup.ResourceGroupName - $RGRBAC = Get-AzRoleAssignment -Scope $ResourceGroup.ResourceId - $RGInfo | Add-Member -MemberType NoteProperty -Name "Type" -Value "Resource Group" -Force - $RGInfo | Add-Member -MemberType NoteProperty -Name "RoleAssignment" -Value $RGRBAC -Force - $RBACDetails += $RGInfo - } - } - $RBACDetails | Export-Csv -Path "RBACDetails.csv" -NoTypeInformation -} - -Get-RBACDetails -ManagementGroup "root" diff --git a/PowerShell/Snippets/Invoke-RESTAzureDevOps.ps1 b/PowerShell/Snippets/Invoke-RESTAzureDevOps.ps1 deleted file mode 100644 index c6e6f5a..0000000 --- a/PowerShell/Snippets/Invoke-RESTAzureDevOps.ps1 +++ /dev/null @@ -1,19 +0,0 @@ -Param( - [string]$org = "x00", - [string]$project = "Modules", - [string]$token = "token" -) - -az devops configure --defaults organization=https://dev.azure.com/$org project=$project -# $repos = az repos list --query "[].[id]" -o table -$repos = az repos list | ConvertFrom-Json - -$base64AuthInfo = [Convert]::ToBase64String([Text.Encoding]::ASCII.GetBytes(("{0}:{1}" -f $user,$token))) -# Disable Repository -$json = '{ "isDisabled" : "false" }' -foreach($repo in $repos){ - $uri = "https://dev.azure.com/$org/$project/_apis/git/repositories/$($repo.id)" + "?api-version=6.0" - Write-Host $repo.name - $result = Invoke-RestMethod -Uri $uri -Method Patch -Body $json -ContentType "application/json" -Headers @{Authorization=("Basic {0}" -f $base64AuthInfo)} - $result -} diff --git a/PowerShell/Snippets/Merge-GitHubPRs.ps1 b/PowerShell/Snippets/Merge-GitHubPRs.ps1 deleted file mode 100644 index 64f5af3..0000000 --- a/PowerShell/Snippets/Merge-GitHubPRs.ps1 +++ /dev/null @@ -1,39 +0,0 @@ -$tfrepos = gh repo list azure -L 5000 --json name --jq '.[].name' | Select-String -Pattern "terraform-azurerm-avm" - -$allItems = @() -Write-Host "Fetching issues and PRs for $($tfrepos.Count) repos" -foreach ($repo in $tfrepos) -{ - Write-Host "Fetching issues and PRs for $repo" - - # Fetch issues - $issues = gh issue list -R "Azure/$repo" --json number,title,url | ConvertFrom-Json - foreach ($issue in $issues) { - $issue | Add-Member -NotePropertyName "repo" -NotePropertyValue $repo - $issue | Add-Member -NotePropertyName "type" -NotePropertyValue "issue" - } - $allItems += $issues - - # Fetch PRs - $prs = gh pr list -R "Azure/$repo" --json number,title,url | ConvertFrom-Json - foreach ($pr in $prs) { - $pr | Add-Member -NotePropertyName "repo" -NotePropertyValue $repo - $pr | Add-Member -NotePropertyName "type" -NotePropertyValue "pr" - } - $allItems += $prs -} - -$allItems | ConvertTo-Json - -# list PRs starting with 'chore' - -$allItems | Where-Object { $_.title -contains "chore: repository governance" -and $_.type -eq "pr" } | Select-Object title,url - -# approve all PRs starting with 'chore' - -foreach ($pr in $allItems | Where-Object { $_.title -eq "chore: repository governance" -and $_.type -eq "pr" }) -{ - Write-Host "Approving PR $($pr.title) on repo $($pr.repo)" - gh pr review $pr.number -R "Azure/$($pr.repo)" --approve --body "Please ensure all checks pass before merging." -# gh pr merge $pr.number -R "Azure/$($pr.repo)" --squash -} diff --git a/PowerShell/Snippets/Pull-Repos.ps1 b/PowerShell/Snippets/Pull-Repos.ps1 deleted file mode 100644 index ccdc3c3..0000000 --- a/PowerShell/Snippets/Pull-Repos.ps1 +++ /dev/null @@ -1,12 +0,0 @@ -Param( - [string]$destinationFolder = "." -) - -$repos = Get-ChildItem -Path $destinationFolder -Directory - -foreach($repo in $repos) { - $($repo.FullName) - Set-Location "$($repo.FullName)"; - git checkout main; - git pull; -} diff --git a/PowerShell/Snippets/RBAC.ps1 b/PowerShell/Snippets/RBAC.ps1 deleted file mode 100644 index a84cd3f..0000000 --- a/PowerShell/Snippets/RBAC.ps1 +++ /dev/null @@ -1,69 +0,0 @@ -# Login to Azure -Connect-AzAccount - -# Function to fetch RBAC details recursively from Management Group to Resource Groups -function Get-RBACHierarchy { - param ( - [string]$ManagementGroupId - ) - - # Fetch Management Group info and RBAC assignments - $mgInfo = Get-AzManagementGroup -GroupId $ManagementGroupId - $mgRBAC = Get-AzRoleAssignment -Scope "/providers/Microsoft.Management/managementGroups/$ManagementGroupId" - - # Print Management Group and RBAC info - Write-Host "Management Group: $($mgInfo.DisplayName)" - foreach ($role in $mgRBAC) { - Write-Host "`tRole: $($role.RoleDefinitionName) - Assigned to: $($role.SignInName)" - } - - # Fetch Subscriptions under the Management Group - $subscriptions = Get-AzSubscription -ManagementGroup $ManagementGroupId - - foreach ($subscription in $subscriptions) { - # Print Subscription info and fetch its RBAC assignments - Write-Host "`tSubscription: $($subscription.Name)" - $subRBAC = Get-AzRoleAssignment -Scope $subscription.Id - - foreach ($role in $subRBAC) { - Write-Host "`t`tRole: $($role.RoleDefinitionName) - Assigned to: $($role.SignInName)" - } - - # Fetch Resource Groups under the Subscription - $resourceGroups = Get-AzResourceGroup -SubscriptionId $subscription.Id - foreach ($rg in $resourceGroups) { - # Print Resource Group info and fetch its RBAC assignments - Write-Host "`t`tResource Group: $($rg.ResourceGroupName)" - $rgRBAC = Get-AzRoleAssignment -ResourceGroupName $rg.ResourceGroupName - - foreach ($role in $rgRBAC) { - Write-Host "`t`t`tRole: $($role.RoleDefinitionName) - Assigned to: $($role.SignInName)" - } - } - } -} - -# Start fetching from the root management group (replace 'root' with your root management group ID if different) -Get-RBACHierarchy -ManagementGroupId "root" - - -function Get-RBACDetails { - param ( - [Parameter(Mandatory=$true)] - [string]$Scope - ) - $rbacDetails = @() - $roleAssignments = Get-AzRoleAssignment -Scope $Scope - foreach ($roleAssignment in $roleAssignments) { - $rbacDetails += [PSCustomObject]@{ - "Scope" = $roleAssignment.Scope - "RoleDefinitionName" = $roleAssignment.RoleDefinitionName - "PrincipalType" = $roleAssignment.PrincipalType - "PrincipalId" = $roleAssignment.PrincipalId - "ObjectId" = $roleAssignment.ObjectId - "ObjectType" = $roleAssignment.ObjectType - "CanDelegate" = $roleAssignment.CanDelegate - } - } - $rbacDetails -} diff --git a/PowerShell/Snippets/Rename-Items.ps1 b/PowerShell/Snippets/Rename-Items.ps1 deleted file mode 100644 index e7c7de9..0000000 --- a/PowerShell/Snippets/Rename-Items.ps1 +++ /dev/null @@ -1,10 +0,0 @@ -# With -Directory for folders and without for files -$WhatIfPreference = 'False' - -$searchprefix = '.terraform' -$a = 'eus' -$b = 'ae' -Get-ChildItem -Path $searchprefix -Recurse | ForEach-Object -Process { Rename-item -Path $_.FullName -NewName ($_.name -replace $a, $b) -Verbose} - -# Remove -Get-ChildItem -Recurse -Hidden | Where-Object { $_.FullName -like '*.terraform*' } | ForEach-Object -Process { Remove-Item -Path $_.FullName -Verbose -Force -Recurse} diff --git a/PowerShell/Snippets/Snippets.psm1 b/PowerShell/Snippets/Snippets.psm1 new file mode 100644 index 0000000..b82a395 --- /dev/null +++ b/PowerShell/Snippets/Snippets.psm1 @@ -0,0 +1,1358 @@ +#Requires -Version 7.0 + +<# +.SYNOPSIS + Reusable automation helpers distilled from the personal snippet collection. + +.DESCRIPTION + Converts the ad-hoc scripts under PowerShell/Snippets into a single module of + advanced functions covering Azure (Az PowerShell, Azure CLI), Azure DevOps + (REST + CLI), GitHub (gh CLI) and local git/file housekeeping. Every function + carries comment-based help, validated parameters in place of the original + hardcoded organisation/subscription/token/prefix literals, and ShouldProcess + support on the state-changing ones. + + Prerequisites depend on the function used: the Az PowerShell modules and an + authenticated context (Connect-AzAccount) for the Az* helpers, the Azure CLI + with the azure-devops extension for the DevOps helpers, and the GitHub CLI + (gh) authenticated for the GitHub helpers. + + Import with: + + Import-Module "$PSScriptRoot/Snippets.psm1" -Force + +.NOTES + Author: Sebastian Gräf + Repo: https://github.com/segraef/Scripts + Version history is tracked in git, not in this header. +#> + +#region Initialisation +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +Import-Module "$PSScriptRoot/../Write-Log.psm1" -Force +#endregion + +#region Functions + +function Test-AvmModule { + <# + .SYNOPSIS + Generate the README and run local Pester/validation/deployment tests for AVM modules. + + .DESCRIPTION + Dot-sources the Set-AVMModule and Test-ModuleLocally tooling from a local + clone of Azure/bicep-registry-modules, regenerates each module README, then + executes the requested end-to-end test cases against the given subscription. + Ensures an Azure context exists and sets the subscription before testing. + + .PARAMETER RepositoryPath + Path to a local clone of Azure/bicep-registry-modules. + + .PARAMETER Module + One or more module paths relative to avm/res (for example 'web/site'). + + .PARAMETER SubscriptionId + Azure subscription ID used as the deployment/validation target context. + + .PARAMETER TenantId + Azure AD tenant ID injected as the TenantId additional token. + + .PARAMETER NamePrefix + Naming prefix injected as the namePrefix additional token. + + .PARAMETER TestCase + Test case folder names under tests/e2e to run. Pass 'all' to discover and + run every test case folder. Defaults to 'all'. + + .PARAMETER Location + Azure region used for validation/deployment. Defaults to australiaeast. + + .EXAMPLE + Test-AvmModule -RepositoryPath ~/git/bicep-registry-modules -Module 'web/site' -SubscriptionId $subId -TenantId $tenantId -NamePrefix 'asf3re' + Regenerates the README for avm/res/web/site and runs all its e2e test cases. + + .OUTPUTS + None. Emits test progress and results to the log/host. + #> + [CmdletBinding()] + param + ( + [Parameter(Mandatory)] + [ValidateNotNullOrEmpty()] + [string]$RepositoryPath, + + [Parameter(Mandatory)] + [ValidateNotNullOrEmpty()] + [string[]]$Module, + + [Parameter(Mandatory)] + [ValidateNotNullOrEmpty()] + [string]$SubscriptionId, + + [Parameter(Mandatory)] + [ValidateNotNullOrEmpty()] + [string]$TenantId, + + [Parameter(Mandatory)] + [ValidateNotNullOrEmpty()] + [string]$NamePrefix, + + [Parameter()] + [ValidateNotNullOrEmpty()] + [string[]]$TestCase = @('all'), + + [Parameter()] + [ValidateNotNullOrEmpty()] + [string]$Location = 'australiaeast' + ) + + try { + if (-not (Get-AzContext)) { + Write-Log 'No Azure context found. Authenticating.' + Connect-AzAccount + } + Set-AzContext -SubscriptionId $SubscriptionId | Out-Null + } + catch { + Write-Log -Message 'Failed to establish Azure context.' -ErrorRecord $_ + throw + } + + try { + . "$RepositoryPath/utilities/tools/Set-AVMModule.ps1" + . "$RepositoryPath/utilities/tools/Test-ModuleLocally.ps1" + } + catch { + Write-Log -Message "Failed to load AVM tooling from '$RepositoryPath'." -ErrorRecord $_ + throw + } + + foreach ($currentModule in $Module) { + Write-Log "Generating README for module '$currentModule'." + try { + Set-AVMModule -ModuleFolderPath "$RepositoryPath/avm/res/$currentModule" -Recurse + } + catch { + Write-Log -Message "Failed to generate README for '$currentModule'." -ErrorRecord $_ + throw + } + + $testModuleLocallyInput = @{ + TemplateFilePath = "$RepositoryPath/avm/res/$currentModule/main.bicep" + PesterTest = $true + ValidationTest = $true + DeploymentTest = $true + ValidateOrDeployParameters = @{ + Location = $Location + SubscriptionId = $SubscriptionId + RemoveDeployment = $true + } + AdditionalTokens = @{ + namePrefix = $NamePrefix + TenantId = $TenantId + } + } + + $resolvedCases = $TestCase + if ($TestCase -contains 'all') { + $resolvedCases = Get-ChildItem -Path "$RepositoryPath/avm/res/$currentModule/tests/e2e" -Directory | + ForEach-Object { $_.Name } + } + + foreach ($case in $resolvedCases) { + Write-Log "Running test case '$case' on module '$currentModule'." + $testModuleLocallyInput.ModuleTestFilePath = "$RepositoryPath/avm/res/$currentModule/tests/e2e/$case/main.test.bicep" + try { + Test-ModuleLocally @testModuleLocallyInput + } + catch { + Write-Log -Message "Test case '$case' on module '$currentModule' failed." -ErrorRecord $_ + } + } + } +} + +function Add-GitHubIssueToProject { + <# + .SYNOPSIS + Add open issues from matching GitHub repositories to a GitHub project. + + .DESCRIPTION + Lists repositories in the given owner whose name matches a pattern, collects + their open issues via the GitHub CLI, and adds each issue to the named + project. Repositories matching the exclude pattern are skipped. + + .PARAMETER Owner + GitHub owner/organisation to list repositories from. + + .PARAMETER RepositoryFilter + Substring used to select repositories by name (for example 'terraform-azurerm-avm'). + + .PARAMETER Project + Title of the GitHub project to add the issues to. + + .PARAMETER ExcludeRepository + Repository name to skip (for example a template repo). Optional. + + .PARAMETER RepositoryLimit + Maximum number of repositories to fetch from the owner. Defaults to 5000. + + .EXAMPLE + Add-GitHubIssueToProject -Owner Azure -RepositoryFilter 'terraform-azurerm-avm' -Project 'AVM - Module Issues' -ExcludeRepository 'terraform-azurerm-avm-template' + Adds every open issue from matching repos to the project, skipping the template repo. + + .OUTPUTS + None. Adds issues to the project as a side effect. + #> + [CmdletBinding(SupportsShouldProcess)] + param + ( + [Parameter(Mandatory)] + [ValidateNotNullOrEmpty()] + [string]$Owner, + + [Parameter(Mandatory)] + [ValidateNotNullOrEmpty()] + [string]$RepositoryFilter, + + [Parameter(Mandatory)] + [ValidateNotNullOrEmpty()] + [string]$Project, + + [Parameter()] + [string]$ExcludeRepository, + + [Parameter()] + [ValidateRange(1, 5000)] + [int]$RepositoryLimit = 5000 + ) + + try { + $repos = gh repo list $Owner -L $RepositoryLimit --json name --jq '.[].name' | + Select-String -Pattern $RepositoryFilter | + ForEach-Object { $_.Line } + } + catch { + Write-Log -Message "Failed to list repositories for owner '$Owner'." -ErrorRecord $_ + throw + } + + Write-Log "Fetching issues for $($repos.Count) repositories." + foreach ($repo in $repos) { + if ($ExcludeRepository -and $repo -eq $ExcludeRepository) { + Write-Log "Skipping excluded repository '$repo'." + continue + } + + Write-Log "Fetching issues for '$repo'." + try { + $issues = gh issue list -R "$Owner/$repo" --json 'number,title,url' | ConvertFrom-Json + } + catch { + Write-Log -Message "Failed to list issues for '$repo'." -ErrorRecord $_ + continue + } + + foreach ($issue in $issues) { + if ($PSCmdlet.ShouldProcess($issue.url, "Add to project '$Project'")) { + Write-Log "Adding issue '$($issue.title)' from '$repo' to project '$Project'." + try { + gh issue edit $issue.url --add-project $Project + } + catch { + Write-Log -Message "Failed to add issue '$($issue.url)' to project '$Project'." -ErrorRecord $_ + } + } + } + } +} + +function Add-AzureDevOpsPullRequest { + <# + .SYNOPSIS + Open a pull request from a source branch across every repository in an Azure DevOps project. + + .DESCRIPTION + Signs in to Azure DevOps with the supplied PAT, configures the default + organisation and project, lists all repositories, and creates a pull request + from the given source branch in each one. Requires the azure-devops Azure CLI + extension (added if missing). + + .PARAMETER Organization + Azure DevOps organisation name (the segment in https://dev.azure.com/). + + .PARAMETER Project + Azure DevOps project name. + + .PARAMETER Token + Personal access token used to authenticate the Azure CLI to Azure DevOps. + + .PARAMETER SourceBranch + Source branch to open each pull request from (for example users/segraef/provider-upgrade). + + .EXAMPLE + Add-AzureDevOpsPullRequest -Organization contoso -Project Modules -Token $pat -SourceBranch 'users/segraef/provider-upgrade' + Opens a PR from the branch in every repository of the Modules project. + + .OUTPUTS + None. Creates pull requests as a side effect. + #> + [CmdletBinding(SupportsShouldProcess)] + param + ( + [Parameter(Mandatory)] + [ValidateNotNullOrEmpty()] + [string]$Organization, + + [Parameter(Mandatory)] + [ValidateNotNullOrEmpty()] + [string]$Project, + + [Parameter(Mandatory)] + [securestring]$Token, + + [Parameter(Mandatory)] + [ValidateNotNullOrEmpty()] + [string]$SourceBranch + ) + + $orgUrl = "https://dev.azure.com/$Organization" + try { + az extension add --name azure-devops + $plainToken = [System.Net.NetworkCredential]::new('', $Token).Password + $plainToken | az devops login --organization $orgUrl + az devops configure --defaults organization=$orgUrl project=$Project + $repos = az repos list | ConvertFrom-Json + } + catch { + Write-Log -Message "Failed to query Azure DevOps repositories for '$Project'." -ErrorRecord $_ + throw + } + + foreach ($repo in $repos) { + if ($PSCmdlet.ShouldProcess($repo.name, "Create pull request from '$SourceBranch'")) { + Write-Log "Creating pull request in '$($repo.name)'." + try { + az repos pr create --repository $repo.name --source-branch $SourceBranch --open --output table + } + catch { + Write-Log -Message "Failed to create pull request in '$($repo.name)'." -ErrorRecord $_ + } + } + } +} + +function Remove-GoneGitBranch { + <# + .SYNOPSIS + Prune local git branches whose upstream is gone across nested repositories. + + .DESCRIPTION + Walks each git repository under the given root, fetches and prunes remote + tracking references, then deletes local branches whose upstream has been + removed (marked ': gone]' by git branch -vv). Returns to the starting + directory when finished. + + .PARAMETER Path + Root directory whose subdirectories are treated as repositories. Defaults to + the current location. + + .EXAMPLE + Remove-GoneGitBranch -Path ~/git/work + Prunes stale local branches in every repository found under the work folder. + + .OUTPUTS + None. Deletes local branches as a side effect. + #> + [CmdletBinding(SupportsShouldProcess)] + param + ( + [Parameter()] + [ValidateNotNullOrEmpty()] + [string]$Path = (Get-Location).Path + ) + + $origin = (Get-Location).Path + try { + $repos = Get-ChildItem -Path $Path -Directory -Recurse + foreach ($repo in $repos) { + Write-Log "Processing repository '$($repo.FullName)'." + Set-Location $repo.FullName + try { + git checkout main + git pull + git remote update origin --prune + $goneBranches = git branch -vv | + Select-String -Pattern ': gone]' | + ForEach-Object { $_.ToString().Trim().Split(' ')[0] } + + foreach ($branch in $goneBranches) { + if ($PSCmdlet.ShouldProcess("$($repo.FullName):$branch", 'Delete local branch')) { + git branch -D $branch + } + } + } + catch { + Write-Log -Message "Failed to prune branches in '$($repo.FullName)'." -ErrorRecord $_ + } + } + } + finally { + Set-Location $origin + } +} + +function New-GitFeatureBranch { + <# + .SYNOPSIS + Create, push and commit a feature branch across nested repositories. + + .DESCRIPTION + For each repository directory under the given root, creates the named branch, + pushes it with upstream tracking, stages all changes, commits them with the + supplied message and pushes the commit. Returns to the starting directory + when finished. + + .PARAMETER Path + Root directory whose subdirectories are treated as repositories. Defaults to + the current directory. + + .PARAMETER BranchName + Name of the branch to create (for example users/segraef/provider-upgrade). + + .PARAMETER CommitMessage + Commit message used for the staged changes. Defaults to 'provider upgrade'. + + .EXAMPLE + New-GitFeatureBranch -Path . -BranchName 'users/segraef/provider-upgrade' -CommitMessage 'provider upgrade' + Creates and pushes the branch in every repository under the current folder. + + .OUTPUTS + None. Creates branches and commits as a side effect. + #> + [CmdletBinding(SupportsShouldProcess)] + param + ( + [Parameter()] + [ValidateNotNullOrEmpty()] + [string]$Path = '.', + + [Parameter(Mandatory)] + [ValidateNotNullOrEmpty()] + [string]$BranchName, + + [Parameter()] + [ValidateNotNullOrEmpty()] + [string]$CommitMessage = 'provider upgrade' + ) + + $origin = (Get-Location).Path + try { + $repos = Get-ChildItem -Path $Path -Directory + foreach ($repo in $repos) { + if ($PSCmdlet.ShouldProcess($repo.FullName, "Create and push branch '$BranchName'")) { + Write-Log "Creating branch '$BranchName' in '$($repo.FullName)'." + Set-Location $repo.FullName + try { + git checkout -b $BranchName + git push --set-upstream origin $BranchName + git add . + git commit -m $CommitMessage + git push + } + catch { + Write-Log -Message "Failed to create/push branch in '$($repo.FullName)'." -ErrorRecord $_ + } + } + } + } + finally { + Set-Location $origin + } +} + +function New-BuildValidationPolicy { + <# + .SYNOPSIS + Create or update build validation branch policies for Azure DevOps repositories. + + .DESCRIPTION + Enumerates repositories, pipelines and existing policy configurations in each + Azure DevOps project (resolved via REST using the supplied PAT), then creates + or updates an optional, manually triggered build validation policy on the + target branch for every repository whose name matches a pipeline. If no + projects are supplied, all projects in the organisation are processed. + + .PARAMETER Organization + Azure DevOps organisation name. + + .PARAMETER Token + Personal access token used for REST and Azure CLI calls. + + .PARAMETER Project + One or more project names to process. When omitted, all projects are discovered. + + .PARAMETER Branch + Target branch the policy applies to. Defaults to main. + + .PARAMETER ApiVersion + Azure DevOps REST API version. Defaults to 6.0. + + .EXAMPLE + New-BuildValidationPolicy -Organization contoso -Token $pat -Project 'Modules' + Creates or updates build validation policies for the Modules project. + + .OUTPUTS + None. Creates or updates branch policies as a side effect. + #> + [CmdletBinding(SupportsShouldProcess)] + param + ( + [Parameter(Mandatory)] + [ValidateNotNullOrEmpty()] + [string]$Organization, + + [Parameter(Mandatory)] + [securestring]$Token, + + [Parameter()] + [string[]]$Project, + + [Parameter()] + [ValidateNotNullOrEmpty()] + [string]$Branch = 'main', + + [Parameter()] + [ValidateNotNullOrEmpty()] + [string]$ApiVersion = '6.0' + ) + + $plainToken = [System.Net.NetworkCredential]::new('', $Token).Password + $env:AZURE_DEVOPS_EXT_PAT = $plainToken + $authHeader = @{ + Authorization = 'Basic ' + [Convert]::ToBase64String([Text.Encoding]::ASCII.GetBytes(":$plainToken")) + } + + function Invoke-AdoCollection { + param + ( + [Parameter(Mandatory)] + [string]$Uri + ) + $encoded = $Uri -replace ' ', '%20' + (Invoke-RestMethod -Uri $encoded -Headers $authHeader).value + } + + try { + if (-not $Project) { + Write-Log 'No project supplied; discovering all projects.' + $projectNames = (Invoke-AdoCollection -Uri "https://dev.azure.com/$Organization/_apis/projects?api-version=$ApiVersion").name + } + else { + $projectNames = $Project + } + } + catch { + Write-Log -Message 'Failed to resolve Azure DevOps projects.' -ErrorRecord $_ + throw + } + + Write-Log "Processing $($projectNames.Count) project(s)." + foreach ($projectName in $projectNames) { + try { + $repos = Invoke-AdoCollection -Uri "https://dev.azure.com/$Organization/$projectName/_apis/git/repositories?api-version=$ApiVersion" + $pipelines = Invoke-AdoCollection -Uri "https://dev.azure.com/$Organization/$projectName/_apis/pipelines?api-version=$ApiVersion" + $buildPolicies = Invoke-AdoCollection -Uri "https://dev.azure.com/$Organization/$projectName/_apis/policy/configurations?api-version=$ApiVersion" + } + catch { + Write-Log -Message "Failed to read configuration for project '$projectName'." -ErrorRecord $_ + continue + } + + Write-Log "Project '$projectName': $($repos.Count) repos, $($pipelines.Count) pipelines, $($buildPolicies.Count) policies." + foreach ($repo in $repos) { + $buildDefinition = $pipelines | Where-Object { $_.name -eq $repo.name } + if (-not $buildDefinition) { + Write-Log "No matching pipeline for repository '$($repo.name)'; skipping." -Level Verbose + continue + } + + $matchingPolicy = $buildPolicies | Where-Object { $_.settings.buildDefinitionId -eq $buildDefinition.id } + if ($matchingPolicy) { + if ($PSCmdlet.ShouldProcess($repo.name, 'Update build validation policy')) { + Write-Log "Updating build validation policy for '$($repo.name)' using pipeline '$($buildDefinition.name)'." + try { + az repos policy build update ` + --id $matchingPolicy.id ` + --blocking $false ` + --branch $Branch ` + --build-definition-id $buildDefinition.id ` + --display-name $repo.name ` + --enabled $true ` + --manual-queue-only $false ` + --queue-on-source-update-only $false ` + --repository-id $repo.id ` + --valid-duration 0 ` + --project $projectName + } + catch { + Write-Log -Message "Failed to update policy for '$($repo.name)'." -ErrorRecord $_ + } + } + } + else { + if ($PSCmdlet.ShouldProcess($repo.name, 'Create build validation policy')) { + Write-Log "Creating build validation policy for '$($repo.name)' using pipeline '$($buildDefinition.name)'." + try { + az repos policy build create ` + --blocking $false ` + --branch $Branch ` + --build-definition-id $buildDefinition.id ` + --display-name $repo.name ` + --enabled $true ` + --manual-queue-only $false ` + --queue-on-source-update-only $false ` + --repository-id $repo.id ` + --valid-duration 0 ` + --project $projectName + } + catch { + Write-Log -Message "Failed to create policy for '$($repo.name)'." -ErrorRecord $_ + } + } + } + } + } +} + +function New-AzServicePrincipalAssignment { + <# + .SYNOPSIS + Create an Azure AD service principal and assign it a role at a scope. + + .DESCRIPTION + Creates a service principal with a generated display name and grants it the + requested role definition at the supplied scope using the Az PowerShell + modules. Requires an authenticated Azure context. + + .PARAMETER SubscriptionId + Subscription ID used to build the default scope when Scope is not supplied. + + .PARAMETER Role + Role definition name to assign. Defaults to Contributor. + + .PARAMETER Scope + Assignment scope. Defaults to the supplied subscription. + + .PARAMETER DisplayName + Service principal display name. Defaults to a generated '-sp' value. + + .EXAMPLE + New-AzServicePrincipalAssignment -SubscriptionId $subId -Role Reader + Creates a service principal and grants it Reader on the subscription. + + .OUTPUTS + The created role assignment object. + #> + [CmdletBinding(SupportsShouldProcess)] + param + ( + [Parameter(Mandatory)] + [ValidateNotNullOrEmpty()] + [string]$SubscriptionId, + + [Parameter()] + [ValidateNotNullOrEmpty()] + [string]$Role = 'Contributor', + + [Parameter()] + [ValidateNotNullOrEmpty()] + [string]$Scope = "/subscriptions/$SubscriptionId", + + [Parameter()] + [ValidateNotNullOrEmpty()] + [string]$DisplayName = "$(Get-Random -Minimum 100000 -Maximum 999999)-sp" + ) + + if ($PSCmdlet.ShouldProcess($Scope, "Create service principal '$DisplayName' with role '$Role'")) { + try { + $sp = New-AzADServicePrincipal -DisplayName $DisplayName + New-AzRoleAssignment -RoleDefinitionName $Role -ServicePrincipalName $sp.AppId -Scope $Scope + } + catch { + Write-Log -Message "Failed to create service principal or role assignment at '$Scope'." -ErrorRecord $_ + throw + } + } +} + +function Remove-ItemByPrefix { + <# + .SYNOPSIS + Recursively remove files or folders matching a name prefix. + + .DESCRIPTION + Finds items (including hidden ones) whose name matches the given prefix + pattern under the current location and removes them recursively. Honours + -WhatIf/-Confirm via ShouldProcess. + + .PARAMETER Prefix + Name pattern to match (for example '.pre*' or '.git'). + + .PARAMETER ExcludeHidden + Exclude hidden items from the search. By default hidden items are included + (matching the original snippet behaviour). + + .EXAMPLE + Remove-ItemByPrefix -Prefix '.pre*' + Removes every item whose name starts with '.pre' beneath the current folder. + + .OUTPUTS + None. Deletes items as a side effect. + #> + [CmdletBinding(SupportsShouldProcess)] + param + ( + [Parameter(Mandatory)] + [ValidateNotNullOrEmpty()] + [string]$Prefix, + + [Parameter()] + [switch]$ExcludeHidden + ) + + try { + $items = Get-ChildItem -Path $Prefix -Recurse -Hidden:(-not $ExcludeHidden) + } + catch { + Write-Log -Message "Failed to enumerate items matching '$Prefix'." -ErrorRecord $_ + throw + } + + foreach ($item in $items) { + if ($PSCmdlet.ShouldProcess($item.FullName, 'Remove item')) { + try { + Remove-Item -Path $item.FullName -Recurse -Force + } + catch { + Write-Log -Message "Failed to remove '$($item.FullName)'." -ErrorRecord $_ + } + } + } +} + +function ConvertTo-TerraformModuleName { + <# + .SYNOPSIS + Convert child item names to dash-separated terraform-azurerm module names. + + .DESCRIPTION + Reads the immediate child items of the given path, lower-cases each base name + and inserts a dash before each upper-case letter, then emits a quoted + 'terraform-azurerm' string per item. Useful for generating module name + lists. + + .PARAMETER Path + Directory whose child items are converted. Defaults to the current directory. + + .EXAMPLE + ConvertTo-TerraformModuleName -Path . + Emits a quoted terraform-azurerm module name for each child item. + + .OUTPUTS + System.String. One quoted module name per child item. + #> + [CmdletBinding()] + param + ( + [Parameter()] + [ValidateNotNullOrEmpty()] + [string]$Path = '.' + ) + + try { + $names = Get-ChildItem -Path $Path + } + catch { + Write-Log -Message "Failed to enumerate items under '$Path'." -ErrorRecord $_ + throw + } + + foreach ($name in $names) { + $converted = ($name.BaseName -replace '(?-i)[A-Z]', '-$&').ToLower() + Write-Output """terraform-azurerm$converted""," + } +} + +function Export-AzProviderReadAction { + <# + .SYNOPSIS + Export all Azure provider read operations to a CSV file. + + .DESCRIPTION + Queries the full set of Azure provider operations, filters to those whose + operation name contains 'read', and exports the operation names to a CSV + file. Requires an authenticated Azure context. + + .PARAMETER Path + Output CSV path. Defaults to Get-AllActions.csv in the current directory. + + .EXAMPLE + Export-AzProviderReadAction -Path ./read-actions.csv + Writes every read operation to read-actions.csv. + + .OUTPUTS + None. Writes a CSV file as a side effect. + #> + [CmdletBinding(SupportsShouldProcess)] + param + ( + [Parameter()] + [ValidateNotNullOrEmpty()] + [string]$Path = 'Get-AllActions.csv' + ) + + if ($PSCmdlet.ShouldProcess($Path, 'Export provider read actions')) { + try { + Get-AzProviderOperation -OperationSearchString '*' | + Where-Object { $_.Operation -like '*read*' } | + Select-Object Operation | + Export-Csv -Path $Path -NoTypeInformation -Force + } + catch { + Write-Log -Message "Failed to export provider read actions to '$Path'." -ErrorRecord $_ + throw + } + } +} + +function Get-RBACDetails { + <# + .SYNOPSIS + Collect RBAC role assignments for a management group hierarchy or a single scope. + + .DESCRIPTION + With -ManagementGroup, recursively walks the management group, its + subscriptions and their resource groups, attaching role assignments to each + node, and optionally exports the result to CSV. With -Scope, returns a flat + list of role assignments for that single scope. Requires an authenticated + Azure context. + + .PARAMETER ManagementGroup + Management group name to walk recursively (hierarchy mode). + + .PARAMETER Scope + A single resource scope to query (flat mode). + + .PARAMETER CsvPath + Optional CSV output path used in hierarchy mode. + + .EXAMPLE + Get-RBACDetails -ManagementGroup root -CsvPath ./rbac.csv + Walks the root management group hierarchy and writes the result to rbac.csv. + + .EXAMPLE + Get-RBACDetails -Scope '/subscriptions/' + Returns the role assignments at the given subscription scope. + + .OUTPUTS + System.Object[]. The collected RBAC detail objects. + #> + [CmdletBinding(DefaultParameterSetName = 'ManagementGroup', SupportsShouldProcess)] + [Diagnostics.CodeAnalysis.SuppressMessageAttribute( + 'PSUseSingularNouns', '', + Justification = 'RBACDetails is the established name of this collection helper.')] + param + ( + [Parameter(Mandatory, ParameterSetName = 'ManagementGroup')] + [ValidateNotNullOrEmpty()] + [string]$ManagementGroup, + + [Parameter(Mandatory, ParameterSetName = 'Scope')] + [ValidateNotNullOrEmpty()] + [string]$Scope, + + [Parameter(ParameterSetName = 'ManagementGroup')] + [ValidateNotNullOrEmpty()] + [string]$CsvPath + ) + + try { + if ($PSCmdlet.ParameterSetName -eq 'Scope') { + $roleAssignments = Get-AzRoleAssignment -Scope $Scope + return $roleAssignments | ForEach-Object { + [PSCustomObject]@{ + Scope = $_.Scope + RoleDefinitionName = $_.RoleDefinitionName + PrincipalType = $_.PrincipalType + PrincipalId = $_.PrincipalId + ObjectId = $_.ObjectId + ObjectType = $_.ObjectType + CanDelegate = $_.CanDelegate + } + } + } + + $rbacDetails = @() + $mgInfo = Get-AzManagementGroup -GroupName $ManagementGroup + $mgRbac = Get-AzManagementGroupRoleAssignment -GroupId $ManagementGroup + $mgInfo | Add-Member -MemberType NoteProperty -Name 'Type' -Value 'Management Group' -Force + $mgInfo | Add-Member -MemberType NoteProperty -Name 'RoleAssignment' -Value $mgRbac -Force + $rbacDetails += $mgInfo + + $subscriptions = Get-AzManagementGroupSubscriptions -GroupId $ManagementGroup + foreach ($subscription in $subscriptions) { + $subRbac = Get-AzRoleAssignment -Scope $subscription.Id + $subscription | Add-Member -MemberType NoteProperty -Name 'Type' -Value 'Subscription' -Force + $subscription | Add-Member -MemberType NoteProperty -Name 'RoleAssignment' -Value $subRbac -Force + $rbacDetails += $subscription + + $resourceGroups = Get-AzResourceGroup -SubscriptionId $subscription.Id + foreach ($resourceGroup in $resourceGroups) { + $rgInfo = Get-AzResourceGroup -Name $resourceGroup.ResourceGroupName + $rgRbac = Get-AzRoleAssignment -Scope $resourceGroup.ResourceId + $rgInfo | Add-Member -MemberType NoteProperty -Name 'Type' -Value 'Resource Group' -Force + $rgInfo | Add-Member -MemberType NoteProperty -Name 'RoleAssignment' -Value $rgRbac -Force + $rbacDetails += $rgInfo + } + } + } + catch { + Write-Log -Message 'Failed to collect RBAC details.' -ErrorRecord $_ + throw + } + + if ($CsvPath) { + if ($PSCmdlet.ShouldProcess($CsvPath, 'Export RBAC details to CSV')) { + try { + $rbacDetails | Export-Csv -Path $CsvPath -NoTypeInformation + } + catch { + Write-Log -Message "Failed to export RBAC details to '$CsvPath'." -ErrorRecord $_ + throw + } + } + } + + Write-Output $rbacDetails +} + +function Get-RBACHierarchy { + <# + .SYNOPSIS + Print the RBAC hierarchy from a management group down to its resource groups. + + .DESCRIPTION + Walks a management group, its subscriptions and their resource groups, + logging each node and its role assignments (role name plus assignee). A + human-readable companion to Get-RBACDetails. Requires an authenticated Azure + context. + + .PARAMETER ManagementGroupId + Management group ID to start from (for example 'root'). + + .EXAMPLE + Get-RBACHierarchy -ManagementGroupId root + Logs the role assignments across the root management group hierarchy. + + .OUTPUTS + None. Writes the hierarchy to the log/host. + #> + [CmdletBinding()] + param + ( + [Parameter(Mandatory)] + [ValidateNotNullOrEmpty()] + [string]$ManagementGroupId + ) + + try { + $mgInfo = Get-AzManagementGroup -GroupId $ManagementGroupId + $mgRbac = Get-AzRoleAssignment -Scope "/providers/Microsoft.Management/managementGroups/$ManagementGroupId" + + Write-Log "Management Group: $($mgInfo.DisplayName)" + foreach ($role in $mgRbac) { + Write-Log " Role: $($role.RoleDefinitionName) - Assigned to: $($role.SignInName)" + } + + $subscriptions = Get-AzSubscription -ManagementGroup $ManagementGroupId + foreach ($subscription in $subscriptions) { + Write-Log " Subscription: $($subscription.Name)" + $subRbac = Get-AzRoleAssignment -Scope $subscription.Id + foreach ($role in $subRbac) { + Write-Log " Role: $($role.RoleDefinitionName) - Assigned to: $($role.SignInName)" + } + + $resourceGroups = Get-AzResourceGroup -SubscriptionId $subscription.Id + foreach ($rg in $resourceGroups) { + Write-Log " Resource Group: $($rg.ResourceGroupName)" + $rgRbac = Get-AzRoleAssignment -ResourceGroupName $rg.ResourceGroupName + foreach ($role in $rgRbac) { + Write-Log " Role: $($role.RoleDefinitionName) - Assigned to: $($role.SignInName)" + } + } + } + } + catch { + Write-Log -Message "Failed to walk RBAC hierarchy for '$ManagementGroupId'." -ErrorRecord $_ + throw + } +} + +function Invoke-AzureDevOpsRest { + <# + .SYNOPSIS + Patch every repository in an Azure DevOps project via the REST API. + + .DESCRIPTION + Configures the Azure CLI defaults, lists the repositories in the given + project, and sends a PATCH request to each repository's Git API with the + supplied JSON body (default re-enables the repository). Authentication uses + basic auth built from the supplied PAT. + + .PARAMETER Organization + Azure DevOps organisation name. + + .PARAMETER Project + Azure DevOps project name. + + .PARAMETER Token + Personal access token used for basic authentication. + + .PARAMETER Body + JSON body sent with each PATCH request. Defaults to '{ "isDisabled" : "false" }'. + + .PARAMETER ApiVersion + Azure DevOps REST API version. Defaults to 6.0. + + .EXAMPLE + Invoke-AzureDevOpsRest -Organization contoso -Project Modules -Token $pat + Re-enables every repository in the Modules project. + + .OUTPUTS + The REST responses from each repository PATCH. + #> + [CmdletBinding(SupportsShouldProcess)] + param + ( + [Parameter(Mandatory)] + [ValidateNotNullOrEmpty()] + [string]$Organization, + + [Parameter(Mandatory)] + [ValidateNotNullOrEmpty()] + [string]$Project, + + [Parameter(Mandatory)] + [securestring]$Token, + + [Parameter()] + [ValidateNotNullOrEmpty()] + [string]$Body = '{ "isDisabled" : "false" }', + + [Parameter()] + [ValidateNotNullOrEmpty()] + [string]$ApiVersion = '6.0' + ) + + $orgUrl = "https://dev.azure.com/$Organization" + $plainToken = [System.Net.NetworkCredential]::new('', $Token).Password + $authHeader = @{ + Authorization = 'Basic ' + [Convert]::ToBase64String([Text.Encoding]::ASCII.GetBytes(":$plainToken")) + } + + try { + az devops configure --defaults organization=$orgUrl project=$Project + $repos = az repos list | ConvertFrom-Json + } + catch { + Write-Log -Message "Failed to list repositories for project '$Project'." -ErrorRecord $_ + throw + } + + foreach ($repo in $repos) { + $uri = "$orgUrl/$Project/_apis/git/repositories/$($repo.id)?api-version=$ApiVersion" + if ($PSCmdlet.ShouldProcess($repo.name, 'Patch repository')) { + Write-Log "Patching repository '$($repo.name)'." + try { + Invoke-RestMethod -Uri $uri -Method Patch -Body $Body -ContentType 'application/json' -Headers $authHeader + } + catch { + Write-Log -Message "Failed to patch repository '$($repo.name)'." -ErrorRecord $_ + } + } + } +} + +function Approve-GitHubPullRequest { + <# + .SYNOPSIS + Approve matching pull requests across GitHub repositories. + + .DESCRIPTION + Lists repositories in the given owner whose name matches a filter, collects + their open pull requests via the GitHub CLI, and approves every PR whose + title matches the supplied title using gh pr review. Optionally adds a review + body. + + .PARAMETER Owner + GitHub owner/organisation to list repositories from. + + .PARAMETER RepositoryFilter + Substring used to select repositories by name. + + .PARAMETER TitleMatch + Exact pull request title to approve. + + .PARAMETER ReviewBody + Optional review comment body. Defaults to a check-before-merge reminder. + + .PARAMETER RepositoryLimit + Maximum number of repositories to fetch. Defaults to 5000. + + .EXAMPLE + Approve-GitHubPullRequest -Owner Azure -RepositoryFilter 'terraform-azurerm-avm' -TitleMatch 'chore: repository governance' + Approves every matching governance PR across the selected repositories. + + .OUTPUTS + None. Submits PR reviews as a side effect. + #> + [CmdletBinding(SupportsShouldProcess)] + param + ( + [Parameter(Mandatory)] + [ValidateNotNullOrEmpty()] + [string]$Owner, + + [Parameter(Mandatory)] + [ValidateNotNullOrEmpty()] + [string]$RepositoryFilter, + + [Parameter(Mandatory)] + [ValidateNotNullOrEmpty()] + [string]$TitleMatch, + + [Parameter()] + [ValidateNotNullOrEmpty()] + [string]$ReviewBody = 'Please ensure all checks pass before merging.', + + [Parameter()] + [ValidateRange(1, 5000)] + [int]$RepositoryLimit = 5000 + ) + + try { + $repos = gh repo list $Owner -L $RepositoryLimit --json name --jq '.[].name' | + Select-String -Pattern $RepositoryFilter | + ForEach-Object { $_.Line } + } + catch { + Write-Log -Message "Failed to list repositories for owner '$Owner'." -ErrorRecord $_ + throw + } + + Write-Log "Fetching pull requests for $($repos.Count) repositories." + foreach ($repo in $repos) { + try { + $prs = gh pr list -R "$Owner/$repo" --json 'number,title,url' | ConvertFrom-Json + } + catch { + Write-Log -Message "Failed to list pull requests for '$repo'." -ErrorRecord $_ + continue + } + + foreach ($pr in $prs | Where-Object { $_.title -eq $TitleMatch }) { + if ($PSCmdlet.ShouldProcess("$repo#$($pr.number)", 'Approve pull request')) { + Write-Log "Approving PR '$($pr.title)' on repository '$repo'." + try { + gh pr review $pr.number -R "$Owner/$repo" --approve --body $ReviewBody + } + catch { + Write-Log -Message "Failed to approve PR '$($pr.url)'." -ErrorRecord $_ + } + } + } + } +} + +function Rename-ItemByPattern { + <# + .SYNOPSIS + Recursively rename items by replacing a substring in their names. + + .DESCRIPTION + Finds items under the given prefix path and renames each one, replacing the + old pattern with the new value in the item name. Honours -WhatIf/-Confirm via + ShouldProcess. + + .PARAMETER Prefix + Path or name prefix to enumerate (for example '.terraform'). + + .PARAMETER OldValue + Substring to replace in each item name. + + .PARAMETER NewValue + Replacement substring. + + .EXAMPLE + Rename-ItemByPattern -Prefix '.terraform' -OldValue 'eus' -NewValue 'ae' + Renames matching items, swapping 'eus' for 'ae' in their names. + + .OUTPUTS + None. Renames items as a side effect. + #> + [CmdletBinding(SupportsShouldProcess)] + param + ( + [Parameter(Mandatory)] + [ValidateNotNullOrEmpty()] + [string]$Prefix, + + [Parameter(Mandatory)] + [ValidateNotNullOrEmpty()] + [string]$OldValue, + + [Parameter(Mandatory)] + [ValidateNotNullOrEmpty()] + [string]$NewValue + ) + + try { + $items = Get-ChildItem -Path $Prefix -Recurse + } + catch { + Write-Log -Message "Failed to enumerate items matching '$Prefix'." -ErrorRecord $_ + throw + } + + foreach ($item in $items) { + $newName = $item.Name -replace $OldValue, $NewValue + if ($newName -eq $item.Name) { + continue + } + if ($PSCmdlet.ShouldProcess($item.FullName, "Rename to '$newName'")) { + try { + Rename-Item -Path $item.FullName -NewName $newName + } + catch { + Write-Log -Message "Failed to rename '$($item.FullName)'." -ErrorRecord $_ + } + } + } +} + +function Start-AzJitAccess { + <# + .SYNOPSIS + Request just-in-time network access to a virtual machine. + + .DESCRIPTION + Resolves the caller's public IP, builds a JIT network access request for the + given VM and port, and invokes Start-AzJitNetworkAccessPolicy against the + named JIT policy for the requested duration. Requires an authenticated Azure + context with Microsoft Defender for Cloud JIT enabled on the VM. + + .PARAMETER VirtualMachineId + Full resource ID of the target virtual machine. + + .PARAMETER JitPolicyResourceId + Full resource ID of the jitNetworkAccessPolicies/default policy for the VM's region. + + .PARAMETER Port + Port to open. Defaults to 3389 (RDP). + + .PARAMETER DurationHours + How long access stays open, in hours. Defaults to 4.8 (0.2 days). + + .PARAMETER SourceAddress + Allowed source address prefix. Defaults to the caller's public IP. + + .EXAMPLE + Start-AzJitAccess -VirtualMachineId $vmId -JitPolicyResourceId $jitId -Port 22 + Opens SSH from the caller's IP to the VM for the default duration. + + .OUTPUTS + The JIT network access policy result object. + #> + [CmdletBinding(SupportsShouldProcess)] + param + ( + [Parameter(Mandatory)] + [ValidateNotNullOrEmpty()] + [string]$VirtualMachineId, + + [Parameter(Mandatory)] + [ValidateNotNullOrEmpty()] + [string]$JitPolicyResourceId, + + [Parameter()] + [ValidateRange(1, 65535)] + [int]$Port = 3389, + + [Parameter()] + [ValidateRange(0.1, 24)] + [double]$DurationHours = 4.8, + + [Parameter()] + [ValidateNotNullOrEmpty()] + [string]$SourceAddress + ) + + try { + if (-not $SourceAddress) { + $SourceAddress = (Invoke-WebRequest -Uri 'http://ifconfig.me/ip').Content + } + $endTime = (Get-Date).AddHours($DurationHours) + $jitPolicy = @( + @{ + id = $VirtualMachineId + ports = @( + @{ + number = $Port + endTimeUtc = "$endTime" + allowedSourceAddressPrefix = @("$SourceAddress") + } + ) + } + ) + } + catch { + Write-Log -Message 'Failed to build JIT access request.' -ErrorRecord $_ + throw + } + + if ($PSCmdlet.ShouldProcess($VirtualMachineId, "Request JIT access on port $Port")) { + try { + Start-AzJitNetworkAccessPolicy -ResourceId $JitPolicyResourceId -VirtualMachine $jitPolicy + } + catch { + Write-Log -Message "Failed to request JIT access for '$VirtualMachineId'." -ErrorRecord $_ + throw + } + } +} + +#endregion + +#region Execution +Export-ModuleMember -Function @( + 'Test-AvmModule', + 'Add-GitHubIssueToProject', + 'Add-AzureDevOpsPullRequest', + 'Remove-GoneGitBranch', + 'New-GitFeatureBranch', + 'New-BuildValidationPolicy', + 'New-AzServicePrincipalAssignment', + 'Remove-ItemByPrefix', + 'ConvertTo-TerraformModuleName', + 'Export-AzProviderReadAction', + 'Get-RBACDetails', + 'Get-RBACHierarchy', + 'Invoke-AzureDevOpsRest', + 'Approve-GitHubPullRequest', + 'Rename-ItemByPattern', + 'Start-AzJitAccess' +) +#endregion diff --git a/PowerShell/Snippets/Start-AzJit.ps1 b/PowerShell/Snippets/Start-AzJit.ps1 deleted file mode 100644 index 052e919..0000000 --- a/PowerShell/Snippets/Start-AzJit.ps1 +++ /dev/null @@ -1,20 +0,0 @@ -$endTime = $(get-date).adddays(0.2) -$ip = (Invoke-WebRequest -uri "http://ifconfig.me/ip").Content -$JitPolicyVm1 = (@{ - id="/subscriptions//resourceGroups/vm-rg/providers/Microsoft.Compute/virtualMachines/cpc"; - ports=(@{ - number=3389; - endTimeUtc="$endTime"; - allowedSourceAddressPrefix=@("$ip")})}) - -$JitPolicyArr=@($JitPolicyVm1) - -Start-AzJitNetworkAccessPolicy -ResourceId "/subscriptions//resourceGroups/vm-rg/providers/Microsoft.Security/locations/australiaeast/jitNetworkAccessPolicies/default" -VirtualMachine $JitPolicyArr - - -$modules = Get-ChildItem | select BaseName -foreach($module in $modules) { - $module = $($module.BaseName) - Write-output "bla import azuredevops_git_repository.$module Modules/$module" - terraform import azuredevops_git_repository.$module Modules/$module -} diff --git a/PowerShell/Update-AdoRepos.ps1 b/PowerShell/Update-AdoRepos.ps1 deleted file mode 100644 index aa95d1c..0000000 --- a/PowerShell/Update-AdoRepos.ps1 +++ /dev/null @@ -1,110 +0,0 @@ -<# -.SYNOPSIS - Script to clone or update Azure DevOps repositories for all projects in an organization. - -.DESCRIPTION - This script clones or updates Azure DevOps repositories for all projects in a specified organization into a specified destination folder. - -.PARAMETER organization - The Azure DevOps organization name. - -.PARAMETER destinationFolder - The folder where the repositories will be cloned or updated. - -.PARAMETER pat - The personal access token for authentication. - -.INPUTS - None - -.OUTPUTS - None - -.NOTES - Version: 1.0 - Author: Sebastian Graef - Creation Date: 22-03-2025 - Purpose/Change: Initial script development - -.EXAMPLE - CloneUpdate-AdoRepos.ps1 -organization "yourOrg" -destinationFolder "C:\Repos" -pat "yourPAT" -#> - -function Update-AdoRepos { - - [CmdletBinding(SupportsShouldProcess)] - param - ( - [Parameter()] - [string]$organization, - [Parameter()] - [string]$destinationFolder, - [Parameter()] - [string]$pat - ) - - # Function to get all projects in the organization - function Get-AdoProjects($organization,$pat) { - $uri = "https://dev.azure.com/$organization/_apis/projects?api-version=6.0" - $response = Invoke-RestMethod -Uri $uri -Headers @{Authorization = "Basic " + [Convert]::ToBase64String([Text.Encoding]::ASCII.GetBytes(":$pat")) } - return $response.value - } - - # Function to get repositories for a given project - function Get-AdoRepositories($organization,$pat,$project) { - $uri = "https://dev.azure.com/$organization/$project/_apis/git/repositories?api-version=6.0" - $uri = $uri -replace " ", "%20" - Write-Output $uri - $response = Invoke-RestMethod -Uri $uri -Headers @{Authorization = "Basic " + [Convert]::ToBase64String([Text.Encoding]::ASCII.GetBytes(":$pat")) } - return $response.value - } - - # Function to clone or update repositories - function CloneOrUpdateRepo($repo, $projectFolder) { - $repoName = $repo.name - $repoUrl = $repo.remoteUrl - $repoFolder = "$projectFolder/$repoName" - - if (-not (Test-Path -Path $repoFolder)) { - if ($PSCmdlet.ShouldProcess("Cloning $($repo.name)")) { - Write-Output "Cloning $($repo.name)" - git clone $repoUrl $repoFolder - } - } else { - if ($PSCmdlet.ShouldProcess("Pulling/Refreshing $($repo.name)")) { - Write-Output "Pulling/Refreshing $($repo.name)" - Set-Location -Path $repoFolder - git checkout main - git pull - Set-Location -Path $projectFolder - } - } - } - - # Main script - Write-Output "Getting projects ..." - $projects = Get-AdoProjects -organization $organization -pat $pat - Write-Output "Found $($projects.Count) projects: $($projects.name)" - foreach ($project in $projects) { - $projectFolder = "$destinationFolder/$($project.name)" - if (-not (Test-Path -Path $projectFolder)) { - if ($PSCmdlet.ShouldProcess("Creating folder $projectFolder")) { - Write-Output "Creating folder $projectFolder" - New-Item -ItemType Directory -Path $projectFolder - } - } - - Write-Output "Getting repos for $($project.name) ..." - $repos = Get-AdoRepositories -organization $organization -pat $pat -project $project.name - Write-Output "Found $($repos.Count) repos: $($repos.name)" - Read-Host "Press Enter to continue" - foreach ($repo in $repos) { - $response = Read-Host "Do you want to clone/update the repo $($repo.name)? (y/n)" - if ($response -eq 'y') { - CloneOrUpdateRepo -repo $repo -projectFolder $projectFolder - } else { - Write-Output "Skipping $($repo.name)" - } - } - } -} diff --git a/PowerShell/Update-GitHubRepos.ps1 b/PowerShell/Update-GitHubRepos.ps1 deleted file mode 100644 index c82c8c4..0000000 --- a/PowerShell/Update-GitHubRepos.ps1 +++ /dev/null @@ -1,82 +0,0 @@ -<# - .SYNOPSIS - Clones or updates GitHub repositories for a specified organization. - - .DESCRIPTION - This script clones or updates GitHub repositories for a specified organization into a specified destination folder. - - .PARAMETER destinationFolder - The folder where the repositories will be cloned or updated. - - .PARAMETER organization - The GitHub organization name. - - .PARAMETER repos - The list of repositories to clone or update. - - .INPUTS - None - - .OUTPUTS - None - - .NOTES - Version: 1.0 - Author: Sebastian Graef - Creation Date: 22-03-2025 - Purpose/Change: Initial script development - - .EXAMPLE - Get-GitHubRepos -destinationFolder "Git/Folder1" -organization "Azure" -repos @("repo1", "repo2") - - .EXAMPLE - $tfrepos = gh repo list azure -L 5000 --json name --jq '.[].name' | Select-String -Pattern "terraform-azurerm-avm" - Get-GitHubRepos -repos $tfrepos -#> - -function Update-GitHubRepos { - - [CmdletBinding(SupportsShouldProcess)] - param - ( - [Parameter()] - [string]$destinationFolder, - [Parameter()] - [string]$organization, - [Parameter()] - [string[]]$repos - ) - - Write-Output "Found $($repos.Count) repositories." - $confirmation = Read-Host "Do you want to proceed with processing these repositories? (y/n)" - if ($confirmation -ne 'y') { - return - } - - foreach ($repo in $repos) { - $repoPath = Join-Path -Path $destinationFolder -ChildPath "$organization/$repo" - If (!(Test-Path -Path $repoPath)) { - if ($PSCmdlet.ShouldProcess("Cloning repository $repo into $repoPath")) { - New-Item -ItemType Directory -Path $repoPath -Force - Set-Location -Path $repoPath - Write-Output "Cloning repository $repo into $repoPath." - git clone "https://github.com/$organization/$repo.git" - } - } else { - Write-Output "Directory $repoPath already exists. Updating only." - if ((Get-ChildItem -Path $repoPath).Count -eq 0) { - if ($PSCmdlet.ShouldProcess("Cloning repository $repo into $repoPath")) { - Write-Output "Cloning repository $repo into $repoPath." - git clone "https://github.com/$organization/$repo.git" - } - } else { - if ($PSCmdlet.ShouldProcess("Pulling latest changes for $repo")) { - Write-Output "Pulling latest changes for $repo." - Set-Location -Path $repoPath - git checkout main - git pull - } - } - } - } -} diff --git a/PowerShell/Update-Repos.ps1 b/PowerShell/Update-Repos.ps1 index 56170fa..a8dc7a6 100644 --- a/PowerShell/Update-Repos.ps1 +++ b/PowerShell/Update-Repos.ps1 @@ -1,262 +1,115 @@ -<# - .SYNOPSIS - Clones or updates GitHub repositories for a specified organization. - - .DESCRIPTION - This script clones or updates GitHub repositories for a specified organization into a specified destination folder. - - .PARAMETER targetFolder - The folder where the repositories will be cloned or updated. - - .PARAMETER organization - The GitHub organization name. - - .PARAMETER repos - The list of repositories to clone or update. - - .INPUTS - None - - .OUTPUTS - None - - .NOTES - Version: 1.0 - Author: Sebastian Graef - Creation Date: 22-03-2025 - Purpose/Change: Initial script development +#Requires -Version 7.0 - .EXAMPLE - Get-GitHubRepos -targetFolder "Git/Folder1" -organization "Azure" -repos @("repo1", "repo2") - - .EXAMPLE - $tfrepos = gh repo list azure -L 5000 --json name --jq '.[].name' | Select-String -Pattern "terraform-azurerm-avm" - Get-GitHubRepos -repos $tfrepos -#> - -function Update-GitHubRepos { - - [CmdletBinding(SupportsShouldProcess)] - param - ( - [Parameter()] - [string]$targetFolder, - [Parameter()] - [string]$organization, - [Parameter()] - [string[]]$repos - ) +<# +.SYNOPSIS + Clone or update GitHub and/or Azure DevOps repositories into a local folder. - Write-Output "Found $($repos.Count) repositories." - $confirmation = Read-Host "Do you want to proceed with processing these repositories? (y/n)" - if ($confirmation -ne 'y') { - return - } +.DESCRIPTION + Thin dispatcher over RepoTools.psm1. Resolves the destination folder to an + absolute path and routes to Update-GitHubRepos, Update-AdoRepos, or both, + depending on -Provider. Missing repositories are cloned; existing ones are + checked out to the default branch and pulled. - foreach ($repo in $repos) { - $repoPath = Join-Path -Path $targetFolder -ChildPath "$organization/$repo" - If (!(Test-Path -Path $repoPath)) { - if ($PSCmdlet.ShouldProcess("Cloning repository $repo into $repoPath")) { - New-Item -ItemType Directory -Path $repoPath -Force - Set-Location -Path $repoPath - Write-Output "Cloning repository $repo into $repoPath." - git clone "https://github.com/$organization/$repo.git" - } - } else { - Write-Output "Directory $repoPath already exists. Updating only." - if ((Get-ChildItem -Path $repoPath).Count -eq 0) { - if ($PSCmdlet.ShouldProcess("Cloning repository $repo into $repoPath")) { - Write-Output "Cloning repository $repo into $repoPath." - git clone "https://github.com/$organization/$repo.git" - } - } else { - if ($PSCmdlet.ShouldProcess("Pulling latest changes for $repo")) { - Write-Output "Pulling latest changes for $repo." - Set-Location -Path $repoPath - git checkout main - git pull - } - } - } - } -} + Requires the git CLI on PATH. The GitHub path needs an organisation and a + repository list; the Azure DevOps path needs an organisation and a personal + access token. +.PARAMETER Provider + Which source to process: GitHub, AzureDevOps, or All (default). -<# -.SYNOPSIS - Script to clone or update Azure DevOps repositories for all projects in an organization. +.PARAMETER DestinationFolder + The root folder repositories are cloned or updated under. Must already exist. -.DESCRIPTION - This script clones or updates Azure DevOps repositories for all projects in a specified organization into a specified destination folder. +.PARAMETER Organization + The GitHub or Azure DevOps organisation name. -.PARAMETER organization - The Azure DevOps organization name. +.PARAMETER Repos + The GitHub repository names to clone or update (required for GitHub/All). -.PARAMETER targetFolder - The folder where the repositories will be cloned or updated. +.PARAMETER Pat + The Azure DevOps personal access token (required for AzureDevOps/All). -.PARAMETER pat - The personal access token for authentication. +.PARAMETER DefaultBranch + The branch to checkout before pulling on existing clones. Defaults to 'main'. .INPUTS - None + None. .OUTPUTS - None + None. -.NOTES - Version: 1.0 - Author: Sebastian Graef - Creation Date: 22-03-2025 - Purpose/Change: Initial script development +.EXAMPLE + ./Update-Repos.ps1 -Provider GitHub -DestinationFolder 'C:/Repos' -Organization 'Azure' -Repos @('bicep','azure-cli') + Clones or updates the two named GitHub repositories under C:/Repos/Azure. .EXAMPLE - CloneUpdate-AdoRepos.ps1 -organization "yourOrg" -targetFolder "C:\Repos" -pat "yourPAT" + $pat = Read-Host -AsSecureString 'PAT' + ./Update-Repos.ps1 -Provider AzureDevOps -DestinationFolder 'C:/Repos' -Organization 'contoso' -Pat $pat + Clones or updates every repository in every project of the contoso organisation. + +.NOTES + Author: Sebastian Gräf + Repo: https://github.com/segraef/Scripts + Version history is tracked in git, not in this header. #> -function Update-AdoRepos { +[CmdletBinding(SupportsShouldProcess)] +param +( + [Parameter()] + [ValidateSet('GitHub', 'AzureDevOps', 'All')] + [string]$Provider = 'All', + + [Parameter(Mandatory)] + [ValidateNotNullOrEmpty()] + [string]$DestinationFolder, + + [Parameter(Mandatory)] + [ValidateNotNullOrEmpty()] + [string]$Organization, - [CmdletBinding(SupportsShouldProcess)] - param - ( [Parameter()] - [string]$organization, + [string[]]$Repos, + [Parameter()] - [string]$targetFolder, + [securestring]$Pat, + [Parameter()] - [string]$pat - ) + [ValidateNotNullOrEmpty()] + [string]$DefaultBranch = 'main' +) - # Function to get all projects in the organization - function Get-AdoProjects($organization,$pat) { - $uri = "https://dev.azure.com/$organization/_apis/projects?api-version=6.0" - $response = Invoke-RestMethod -Uri $uri -Headers @{Authorization = "Basic " + [Convert]::ToBase64String([Text.Encoding]::ASCII.GetBytes(":$pat")) } - return $response.value - } +#region Initialisation +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' - # Function to get repositories for a given project - function Get-AdoRepositories($organization,$pat,$project) { - $uri = "https://dev.azure.com/$organization/$project/_apis/git/repositories?api-version=6.0" - $uri = $uri -replace " ", "%20" - Write-Output $uri - $response = Invoke-RestMethod -Uri $uri -Headers @{Authorization = "Basic " + [Convert]::ToBase64String([Text.Encoding]::ASCII.GetBytes(":$pat")) } - return $response.value - } +Import-Module "$PSScriptRoot/Write-Log.psm1" -Force +Import-Module "$PSScriptRoot/RepoTools.psm1" -Force +#endregion - # Function to clone or update repositories - function CloneOrUpdateRepo($repo, $projectFolder) { - $repoName = $repo.name - $repoUrl = $repo.remoteUrl - $repoFolder = "$projectFolder/$repoName" +#region Execution +Write-Log "Executing $($MyInvocation.MyCommand.Name) (provider '$Provider')." - if (-not (Test-Path -Path $repoFolder)) { - if ($PSCmdlet.ShouldProcess("Cloning $($repo.name)")) { - Write-Output "Cloning $($repo.name)" - git clone $repoUrl $repoFolder - } - } else { - if ($PSCmdlet.ShouldProcess("Pulling/Refreshing $($repo.name)")) { - Write-Output "Pulling/Refreshing $($repo.name)" - Set-Location -Path $repoFolder - git checkout main - git pull - Set-Location -Path $projectFolder - } - } - } +if (-not (Test-Path -Path $DestinationFolder)) { + throw "The DestinationFolder '$DestinationFolder' does not exist." +} +$DestinationFolder = (Resolve-Path -Path $DestinationFolder).Path - # Main script - Write-Output "Getting projects ..." - $projects = Get-AdoProjects -organization $organization -pat $pat - Write-Output "Found $($projects.Count) projects: $($projects.name)" - foreach ($project in $projects) { - $projectFolder = "$targetFolder/$($project.name)" - if (-not (Test-Path -Path $projectFolder)) { - if ($PSCmdlet.ShouldProcess("Creating folder $projectFolder")) { - Write-Output "Creating folder $projectFolder" - New-Item -ItemType Directory -Path $projectFolder - } +if ($Provider -in 'GitHub', 'All') { + if (-not $Repos) { + throw "The Repos parameter is required for the '$Provider' provider." } + Write-Log 'Updating GitHub repositories.' + Update-GitHubRepos -TargetFolder $DestinationFolder -Organization $Organization -Repos $Repos -DefaultBranch $DefaultBranch +} - Write-Output "Getting repos for $($project.name) ..." - $repos = Get-AdoRepositories -organization $organization -pat $pat -project $project.name - Write-Output "Found $($repos.Count) repos: $($repos.name)" - Read-Host "Press Enter to continue" - foreach ($repo in $repos) { - $response = Read-Host "Do you want to clone/update the repo $($repo.name)? (y/n)" - if ($response -eq 'y') { - CloneOrUpdateRepo -repo $repo -projectFolder $projectFolder - } else { - Write-Output "Skipping $($repo.name)" - } +if ($Provider -in 'AzureDevOps', 'All') { + if (-not $Pat) { + throw "The Pat parameter is required for the '$Provider' provider." } - } + Write-Log 'Updating Azure DevOps repositories.' + Update-AdoRepos -Organization $Organization -TargetFolder $DestinationFolder -Pat $Pat -DefaultBranch $DefaultBranch } -<# -.SYNOPSIS - Script to clone or update repositories for a specified organization. - -.DESCRIPTION - This script clones or updates repositories for a specified organization into a specified destination folder. It supports both GitHub and Azure DevOps repositories. - -.PARAMETER destinationFolder - The folder where the repositories will be cloned or updated. - -.PARAMETER organization - The organization name (GitHub or Azure DevOps). - -.PARAMETER repos - The list of GitHub repositories to clone or update. - -.PARAMETER pat - The personal access token for Azure DevOps authentication. - -.INPUTS - None - -.OUTPUTS - None - -.NOTES - Version: 1.0 - Author: Sebastian Graef - Creation Date: 22-03-2025 - Purpose/Change: Initial script development - -.EXAMPLE - Update-Repos -destinationFolder "C:\Repos" -organization "yourOrg" -repos @("repo1", "repo2") - -.EXAMPLE - Update-Repos -destinationFolder "C:\Repos" -organization "yourOrg" -pat "yourPAT" -#> - -function Update-Repos { - - [CmdletBinding(SupportsShouldProcess)] - param - ( - [Parameter()] - [string]$destinationFolder, - [Parameter()] - [string]$organization, - [Parameter()] - [string[]]$repos, - [Parameter()] - [string]$pat - ) - - # Resolve destinationFolder to an absolute path - if (-not $destinationFolder) { - throw "The destinationFolder parameter cannot be empty." - } - $destinationFolder = Resolve-Path -Path $destinationFolder | ForEach-Object { $_.Path } - - if ($pat) { - Write-Output "- Updating Azure DevOps repositories" - Update-AdoRepos -organization $organization -targetFolder $destinationFolder -pat $pat - } else { - Write-Output "- Updating GitHub repositories" - Update-GitHubRepos -targetFolder $destinationFolder -organization $organization -repos $repos - } -} +Write-Log "Finished $($MyInvocation.MyCommand.Name)." +#endregion diff --git a/PowerShell/Write-Log.ps1 b/PowerShell/Write-Log.ps1 deleted file mode 100644 index 91cbd88..0000000 --- a/PowerShell/Write-Log.ps1 +++ /dev/null @@ -1,71 +0,0 @@ -#Requires -Version 5.1 - -<# -.SYNOPSIS - - -.DESCRIPTION - - -.PARAMETER - - -.INPUTS - - -.OUTPUTS - .log> - -.NOTES - Version: 1.0 - Author: Sebastian Gräf - Creation Date: 21/06/2021 - Purpose/Change: Initial script development - -.EXAMPLE - -#> - -#region Parameters - -[CmdletBinding()] -param -( - [String]$Message, - [Switch]$Warning, - [System.Management.Automation.ErrorRecord]$ErrorObj -) - -#endregion - -#region Initialisations - -$ErrorActionPreference = "Continue" -$VerbosePreference = "Continue" - -#endregion - -#region Declarations -#endregion - -#region Functions -#endregion - -#region Execution - -$LogMessage = "[$(Get-Date -f g)] " - -if ($PSBoundParameters.ContainsKey("ErrorObj")) { - $LogMessage += " $ErrorObj $($ErrorObj.ScriptStackTrace.Split("`n") -join ' <-- ')" - Write-Error -Message $LogMessage -} -elseif ($PSBoundParameters.ContainsKey("Warning")) { - $LogMessage += " $Message" - Write-Warning -Message $LogMessage -} -else { - $LogMessage += " $Message" - Write-Verbose -Message $LogMessage -} - -#endregion diff --git a/PowerShell/Write-Log.psm1 b/PowerShell/Write-Log.psm1 new file mode 100644 index 0000000..741f540 --- /dev/null +++ b/PowerShell/Write-Log.psm1 @@ -0,0 +1,82 @@ +#Requires -Version 7.0 + +<# +.SYNOPSIS + Structured, timestamped logging for scripts in this repository. + +.DESCRIPTION + Provides a single Write-Log command that writes a timestamped, level-tagged + line to the appropriate PowerShell stream. Import it from a script with: + + Import-Module "$PSScriptRoot/Write-Log.psm1" -Force + + This is a module (.psm1), not a script: importing it actually defines the + Write-Log command, unlike importing a plain .ps1. + +.NOTES + Author: Sebastian Gräf + Repo: https://github.com/segraef/Scripts +#> + +function Write-Log { + <# + .SYNOPSIS + Write a timestamped, level-tagged log line. + + .PARAMETER Message + The text to log. Accepts pipeline input. + + .PARAMETER Level + Severity: Info (default), Warning, Error, or Verbose. Selects the stream. + + .PARAMETER ErrorRecord + An ErrorRecord to append (message + collapsed stack trace). Forces Error level. + + .EXAMPLE + Write-Log 'Starting deployment.' + + .EXAMPLE + try { ... } catch { Write-Log -Message 'Deployment failed.' -ErrorRecord $_ } + #> + [CmdletBinding()] + [Diagnostics.CodeAnalysis.SuppressMessageAttribute( + 'PSAvoidUsingWriteHost', '', + Justification = 'Info-level output is intended for the interactive console.')] + [Diagnostics.CodeAnalysis.SuppressMessageAttribute( + 'PSAvoidOverwritingBuiltInCmdlets', '', + Justification = 'Write-Log is this repository deliberate logging helper, not the obscure built-in.')] + param + ( + [Parameter(Mandatory, Position = 0, ValueFromPipeline)] + [ValidateNotNull()] + [string]$Message, + + [Parameter()] + [ValidateSet('Info', 'Warning', 'Error', 'Verbose')] + [string]$Level = 'Info', + + [Parameter()] + [System.Management.Automation.ErrorRecord]$ErrorRecord + ) + + process { + $timestamp = Get-Date -Format 'yyyy-MM-dd HH:mm:ss' + $line = "[$timestamp] $Message" + + if ($ErrorRecord) { + $traceText = "$($ErrorRecord.ScriptStackTrace)" + $trace = ($traceText -split "`n" | Where-Object { $_ }) -join ' <- ' + Write-Error -Message "$line | $($ErrorRecord.Exception.Message) | $trace" + return + } + + switch ($Level) { + 'Warning' { Write-Warning $line } + 'Error' { Write-Error $line } + 'Verbose' { Write-Verbose $line } + default { Write-Host $line } + } + } +} + +Export-ModuleMember -Function Write-Log diff --git a/PowerShell/_Template.ps1 b/PowerShell/_Template.ps1 index 993feff..5f9973d 100644 --- a/PowerShell/_Template.ps1 +++ b/PowerShell/_Template.ps1 @@ -1,91 +1,76 @@ -#Requires -Version 5.1 +#Requires -Version 7.0 <# .SYNOPSIS - + One-line summary of what this script does. .DESCRIPTION - + Longer description: what it automates, its prerequisites (modules, CLIs, + authentication) and any side effects it has on the environment. -.PARAMETER - +.PARAMETER Name + Describe each parameter. Repeat this section for every parameter. .INPUTS - + None. Or describe the objects accepted from the pipeline. .OUTPUTS - .log> - -.NOTES - Version: 1.0 - Author: - Creation Date: - Purpose/Change: Initial script development + None. Or describe the objects / files produced. .EXAMPLE - -#> + ./Verb-Noun.ps1 -Name 'value' + Describe what this example does. -#region Parameters +.NOTES + Author: Sebastian Gräf + Repo: https://github.com/segraef/Scripts + Version history is tracked in git, not in this header. +#> -[CmdletBinding()] +[CmdletBinding(SupportsShouldProcess)] param ( - [Parameter()] - [String]$String, - [Parameter()] - [SecureString]$SecureString -) + [Parameter(Mandatory)] + [ValidateNotNullOrEmpty()] + [string]$Name, -#endregion - -#region Initialisations - -$ErrorActionPreference = "Continue" -$VerbosePreference = "Continue" - -Import-Module ..\Write-Log.ps1 + [Parameter()] + [securestring]$Secret +) -#endregion +#region Initialisation +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' -#region Declarations +Import-Module "$PSScriptRoot/Write-Log.psm1" -Force #endregion #region Functions - -function FunctionName { - Param() - - begin { - Write-Log "Let's start !" - } - - process { - try { - Write-Output "Hello Template !" - } - - catch { - Write-Output $_ - Write-Log $_ -Warning - } - } - - end { - if ($?) { - Write-Log "Completed successfully !" +function Invoke-Example { + [CmdletBinding(SupportsShouldProcess)] + param + ( + [Parameter(Mandatory)] + [ValidateNotNullOrEmpty()] + [string]$Name + ) + + Write-Log "Processing '$Name'." + + if ($PSCmdlet.ShouldProcess($Name, 'Process')) { + try { + Write-Output "Hello, $Name." + } + catch { + Write-Log -Message "Failed to process '$Name'." -ErrorRecord $_ + throw + } } - } } - #endregion #region Execution - -Write-Log "Executing $($MyInvocation.MyCommand.Name)" - -FunctionName -String $String -SecureString $SecureString - -Write-Log "Finished executing $($MyInvocation.MyCommand.Name)" - +Write-Log "Executing $($MyInvocation.MyCommand.Name)." +Invoke-Example -Name $Name +Write-Log "Finished $($MyInvocation.MyCommand.Name)." #endregion diff --git a/Python/hibp.py b/Python/hibp.py index 28268ac..3ae27eb 100644 --- a/Python/hibp.py +++ b/Python/hibp.py @@ -1,4 +1,18 @@ -import os, json, hashlib, requests +"""Command-line Have I Been Pwned (HIBP) breach checker. + +Prompts for an email address and a password, then queries the HIBP API to +report whether the email appears in known data breaches and whether the +password appears in the Pwned Passwords range API. Finally it lists every +breach currently tracked by HIBP. + +The HIBP API key is read from the HIBP_API_KEY environment variable (via a +.env file). The script exits with an error if the key is not set. +""" +import os +import hashlib +import getpass + +import requests from dotenv import load_dotenv # Load the environment variables from .env @@ -6,42 +20,83 @@ # Get the API key from the environment variables API_KEY = os.getenv('HIBP_API_KEY') +if not API_KEY: + raise RuntimeError('HIBP_API_KEY is not set. Add it to your environment or .env file.') + +API_URL = 'https://haveibeenpwned.com/api/v3' +PWD_API_URL = 'https://api.pwnedpasswords.com/range' + +# Request timeout (seconds) so a stalled network call doesn't hang the CLI. +REQUEST_TIMEOUT = 10 + +# HIBP requires both an API key and a descriptive User-Agent. +HIBP_HEADERS = { + 'hibp-api-key': API_KEY, + 'User-Agent': 'segraef-Scripts-hibp-checker', +} + +# The Pwned Passwords range API is a separate, unauthenticated service: +# never send the HIBP API key to it. +PWD_HEADERS = {'User-Agent': 'segraef-Scripts-hibp-checker'} + + +def check_email_breach(email): + """Check whether an email address appears in known data breaches. + + Args: + email: The email address to look up. + """ + try: + response = requests.get( + f'{API_URL}/breachedaccount/{email}', + headers=HIBP_HEADERS, + timeout=REQUEST_TIMEOUT, + ) + except requests.RequestException as exc: + print(f"Error checking email: {exc}") + return + + # Check the status code of the response + if response.status_code == 404: + print("Email not found in data breaches") + elif response.status_code != 200: + print("Error checking email") + else: + # Extract the name of the breaches from the response + breaches = [breach['Name'] for breach in response.json()] + print(f"Email found in following breaches: {', '.join(breaches)}.") + + +def check_password_breach(password): + """Check whether a password appears in the Pwned Passwords range API. + + The password is SHA-1 hashed locally and only the first five characters of + the hash are sent to the API (k-anonymity), so the plaintext never leaves + this machine. + + Args: + password: The plaintext password to check. + """ + # Hash the password before sending it to the HIBP API + hashed_password = hashlib.sha1(password.encode('utf-8')).hexdigest().upper() + prefix = hashed_password[:5] + suffix = hashed_password[5:] + + try: + response = requests.get( + f'{PWD_API_URL}/{prefix}', + headers=PWD_HEADERS, + timeout=REQUEST_TIMEOUT, + ) + except requests.RequestException as exc: + print(f"Error checking password: {exc}") + return + + # Check the status code of the response + if response.status_code != 200: + print("Error checking password") + return -api_url = 'https://haveibeenpwned.com/api/v3' -pwd_api_url = 'https://api.pwnedpasswords.com/range' - -# Get the email address from the user input -email = input("Enter your email address: ") - -# Use the API key in the headers -headers = {'hibp-api-key': API_KEY} - -# Send the GET request to the HIBP API -response = requests.get(f'{api_url}/breachedaccount/{email}', headers=headers) - -# Check the status code of the response -if response.status_code == 404: - print("Email not found in data breaches") -elif response.status_code != 200: - print("Error checking email") -else: - # Extract the name of the breaches from the response - breaches = [breach['Name'] for breach in response.json()] - print(f"Email found in following breaches: {', '.join(breaches)}.") - -# Hash the password before sending it to the HIBP API -password = input("Enter your password: ") -hashed_password = hashlib.sha1(password.encode('utf-8')).hexdigest().upper() -prefix = hashed_password[:5] -suffix = hashed_password[5:] - -# Send the GET request to the HIBP API -response = requests.get(f'{pwd_api_url}/{prefix}', headers=headers) - -# Check the status code of the response -if response.status_code != 200: - print("Error checking password") -else: # Check if the hashed password suffix exists in the response for line in response.text.splitlines(): line_suffix, count = line.split(':') @@ -49,24 +104,48 @@ print(f"Password found {count} times. Please use a different password.") break else: - print(f"Password not found. You can use this password.") - -# Make the GET request -response = requests.get(f'{api_url}/breaches', headers=headers) - -# Print the status code of the response -print(response.status_code) - -# Print the response content (a list of breach objects) -# print(response.json()) - -# Display the breaches -breaches = json.loads(response.text) -count = len(breaches) -for breach in breaches: - print(f'Name: {breach["Name"]}') - print(f'Title: {breach["Title"]}') - print(f'Domain: {breach["Domain"]}') - print(f'Breach date: {breach["BreachDate"]}') - print('---') -print(f'Total number of breaches: {count}') + print("Password not found. You can use this password.") + + +def list_all_breaches(): + """Fetch and print every breach currently tracked by HIBP.""" + try: + response = requests.get( + f'{API_URL}/breaches', + headers=HIBP_HEADERS, + timeout=REQUEST_TIMEOUT, + ) + except requests.RequestException as exc: + print(f"Error fetching breaches: {exc}") + return + + if response.status_code != 200: + print(f"Error fetching breaches (HTTP {response.status_code}).") + return + + # Display the breaches + breaches = response.json() + count = len(breaches) + for breach in breaches: + print(f'Name: {breach["Name"]}') + print(f'Title: {breach["Title"]}') + print(f'Domain: {breach["Domain"]}') + print(f'Breach date: {breach["BreachDate"]}') + print('---') + print(f'Total number of breaches: {count}') + + +def main(): + """Run the interactive email and password breach checks.""" + # Get the email address from the user input + email = input("Enter your email address: ") + check_email_breach(email) + + password = getpass.getpass("Enter your password: ") + check_password_breach(password) + + list_all_breaches() + + +if __name__ == '__main__': + main() diff --git a/Python/requirements.txt b/Python/requirements.txt index f229360..f522632 100644 --- a/Python/requirements.txt +++ b/Python/requirements.txt @@ -1 +1,3 @@ -requests +requests>=2.32,<3 +flask +python-dotenv diff --git a/README.md b/README.md index 5cb68ff..fcbfaff 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,25 @@ # ![AzureIcon] ![BashIcon] ![PowershellIcon] Scripts -This repository contains Azure and Windows PowerShell scripts for developers and administrators to develop, deploy, and manage Microsoft Azure and Microsoft Windows ecosystem. +A collection of automation scripts across PowerShell, Bash, Python and JavaScript for Azure, DevOps and general infrastructure tasks. + +## Repository structure + +| Path | Contents | +|---|---| +| [`PowerShell/`](PowerShell/) | Azure / DevOps automation scripts. Start from [`_Template.ps1`](PowerShell/_Template.ps1); shared logging is [`Write-Log.psm1`](PowerShell/Write-Log.psm1). | +| [`PowerShell/Snippets/`](PowerShell/Snippets/) | Smaller, single-purpose helpers. | +| [`Bash/`](Bash/) | Shell utilities. Start from [`_Template.sh`](Bash/_Template.sh); shared logging is [`log.sh`](Bash/log.sh). | +| [`Python/`](Python/) | Python utilities (HIBP breach checks, helpers). | +| [`flask/`](flask/) | Small Flask web app wrapping the HIBP checks. | +| [`JavaScript/`](JavaScript/), [`REST/`](REST/), [`graphql/`](graphql/) | Browser snippet, `.http` request samples, GraphQL queries. | +| [`cheatsheets/`](cheatsheets/) | Quick-reference notes (git, python, rhel). | +| [`PSScriptAnalyzerSettings.psd1`](PSScriptAnalyzerSettings.psd1) | Lint/format rules enforced by CI. | + +This repository was generated from [segraef/Template](https://github.com/segraef/Template) and stays in sync with it via the [Template Sync](.github/workflows/template-sync.yml) workflow. ## Status -[![SuperLinter]()]() +[![SuperLinter]()]() [![PowerShell ScriptAnalyzer]()](https://github.com/segraef/Scripts/actions/workflows/scriptanalyzer.yml) @@ -32,21 +47,12 @@ If you would like to become an active contributor to this repository or project, -[ProjectSetup]: -[CreateFromTemplate]: -[GitHubDocs]: -[AzureDevOpsDocs]: -[GitHubIssues]: +[GitHubIssues]: [Contributing]: CONTRIBUTING.md [AzureIcon]: docs/media/MicrosoftAzure-32px.png [PowershellIcon]: docs/media/MicrosoftPowerShellCore-32px.png [BashIcon]: docs/media/Bash_Logo_black_and_white_icon_only-32px.svg.png - -[Az]: -[AzGallery]: -[PowerShellCore]: - [MicrosoftAzureDocs]: [PowerShellDocs]: diff --git a/_in progress/.pipelines/.templates/pipeline.artifacts.yml b/_in progress/.pipelines/.templates/pipeline.artifacts.yml deleted file mode 100644 index 614af58..0000000 --- a/_in progress/.pipelines/.templates/pipeline.artifacts.yml +++ /dev/null @@ -1,66 +0,0 @@ -variables: - # Artifacts Feed - artifactFeedPath: '$(System.Teamproject)/ResourceModules' - packagePath: ModulePackages - downloadDirectory: $(Build.SourcesDirectory)/$(packagePath) - - # Artifacts - RGModuleName: 'Microsoft.resources.resourcegroups' - RGModuleVersion: '*' - - VnetModuleName: 'Microsoft.Network.virtualnetworks' - VnetModuleVersion: '*' - - VnetPeeringModuleName: 'Microsoft.Network.VirtualNetworks.VirtualNetworkPeering' - VnetPeeringModuleVersion: '*' - - RouteTablesModuleName: 'Microsoft.Network.RouteTables' - RouteTablesModuleVersion: '*' - - NsgModuleName: 'Microsoft.Network.networksecuritygroups' - NsgModuleVersion: '*' - - AzureBastionModuleName: 'Microsoft.Network.bastionhosts' - AzureBastionModuleVersion: '*' - - DdosProtectionPlansModuleName: 'Microsoft.Network.DdosProtectionPlans' - DdosProtectionPlansModuleVersion: '*' - - ExpressRouteCircuitModuleName: 'Microsoft.Network.ExpressRouteCircuits' - ExpressRouteCircuitModuleVersion: '*' - - PublicIpPrefixesModuleName: 'Microsoft.Network.PublicIpPrefixes' - PublicIpPrefixesModuleVersion: '*' - - VirtualNetworkGatewayModuleName: 'Microsoft.Network.VirtualNetworkGateways' - VirtualNetworkGatewayModuleVersion: '*' - - VirtualNetworkGatewayConnectionModuleName: 'Microsoft.Network.vpnGateways.connections' - VirtualNetworkGatewayConnectionModuleVersion: '*' - - LocalNetworkGatewayModuleName: 'Microsoft.Network.LocalNetworkGateways' - LocalNetworkGatewayModuleVersion: '*' - - NetworkWatcherModuleName: 'Microsoft.Network.networkWatchers' - NetworkWatcherModuleVersion: '0.4.1033-prerelease' - - StorageAccountsModuleName: 'Microsoft.Storage.storageAccounts' - StorageAccountsModuleVersion: '*' - - KVModuleName: 'Microsoft.keyvault.vaults' - KVModuleVersion: '*' - - VMModuleName: 'Microsoft.compute.virtualmachines' - VMModuleVersion: '0.4.1024-prerelease' - - PEModuleName: 'microsoft.network.privateendpoints' - PEModuleVersion: '*' - - SQLMIModuleName: 'Microsoft.sql.managedinstances' - SQLMIModuleVersion: '0.4.1025-prerelease' - - RSVModuleName: 'Microsoft.recoveryservices.vaults' - RSVModuleVersion: '0.4.1032-prerelease' - - LogAnalyticsModuleName: 'Microsoft.OperationalInsights.workspaces' - LogAnalyticsModuleVersion: '*' diff --git a/_in progress/.pipelines/.templates/pipeline.jobs.artifact.deploy.yml b/_in progress/.pipelines/.templates/pipeline.jobs.artifact.deploy.yml deleted file mode 100644 index 4408895..0000000 --- a/_in progress/.pipelines/.templates/pipeline.jobs.artifact.deploy.yml +++ /dev/null @@ -1,144 +0,0 @@ -parameters: - jobName: - moduleName: - moduleVersion: - parameterFilePath: - dependsOn: [] - environment: '' - timeoutInMinutes: 90 - artifactFeedPath: '$(artifactFeedPath)' - serviceConnection: '$(serviceConnection)' - vmImage: $(vmImage) - poolName: $(poolName) - location: '$(location)' - resourceGroupName: '$(resourceGroupName)' - managementGroupId: '$(managementGroupId)' - displayName: 'Deploy module' - whatif: false - enabled: true - -jobs: - - deployment: ${{ parameters.jobName }}${{ parameters.whatif }} - ${{ if eq( parameters.whatif, true) }}: - displayName: ${{ parameters.displayName }} WhatIf - ${{ if ne( parameters.whatif, true) }}: - displayName: ${{ parameters.displayName }} - ${{ if ne( parameters.dependsOn, '') }}: - dependsOn: - - ${{ each dependency in parameters.dependsOn }}: - - ${{ dependency }}${{ parameters.whatif }} - environment: ${{ parameters.environment }} - timeoutInMinutes: ${{ parameters.timeoutInMinutes }} - pool: - ${{ if ne(parameters.vmImage, '') }}: - vmImage: ${{ parameters.vmImage }} - ${{ if ne(parameters.poolName, '') }}: - name: ${{ parameters.poolName }} - strategy: - runOnce: - deploy: - steps: - - checkout: self - - powershell: | - $lowerModuleName = "${{ parameters.moduleName }}".ToLower() - Write-Host "##vso[task.setVariable variable=lowerModuleName]$lowerModuleName" - displayName: 'Prepare download from artifacts feed' - enabled: ${{ parameters.enabled }} - - - task: UniversalPackages@0 - displayName: 'Download module [${{ parameters.moduleName }}] version [${{ parameters.moduleVersion }}] from feed [${{ parameters.artifactFeedPath }}]' - inputs: - command: download - vstsFeed: '${{ parameters.artifactFeedPath }}' - vstsFeedPackage: '$(lowerModuleName)' - vstsPackageVersion: '${{ parameters.moduleVersion }}' - downloadDirectory: '$(downloadDirectory)/$(lowerModuleName)' - enabled: ${{ parameters.enabled }} - - - task: AzurePowerShell@5 - displayName: 'Deploy module [${{ parameters.moduleName }}] version [${{ parameters.moduleVersion }}] in [${{ parameters.resourcegroupname }}] via [${{ parameters.serviceConnection }}]' - name: DeployResource - inputs: - azureSubscription: ${{ parameters.serviceConnection }} - errorActionPreference: stop - azurePowerShellVersion: LatestVersion - ScriptType: InlineScript - failOnStandardError: true - pwsh: true - inline: | - $templateFilePath = Join-Path "$(downloadDirectory)/${{ parameters.moduleName }}" 'deploy.json' - Write-Verbose "Got path: $templateFilePath" -Verbose - Write-Verbose "downloadDirectory: $(downloadDirectory)" -Verbose - Get-ChildItem $(downloadDirectory) -recurse - - $whatIf = [System.Convert]::ToBoolean('${{ parameters.whatif }}') - $moduleName = "${{ parameters.moduleName }}" - $moduleName[0..20] -join "" - - $moduleName = "${{ parameters.moduleName }}" - $moduleName = $moduleName[0..20] -join "" - $deploymentName = "$moduleName-$(-join (Get-Date -Format yyyyMMddTHHMMssffffZ)[0..43])" - $deploymentName - - $DeploymentInputs = @{ - DeploymentName = "$moduleName-$(-join (Get-Date -Format yyyyMMddTHHMMssffffZ)[0..43])" - - TemplateFile = $templateFilePath - TemplateParameterFile = "$(Build.SourcesDirectory)/$(environmentPath)/${{ parameters.parameterFilePath }}" - Verbose = $true - ErrorAction = "Stop" - whatIf = $whatIf - } - - $deploymentSchema = (ConvertFrom-Json (Get-Content -Raw -Path $templateFilePath)).'$schema' - switch -regex ($deploymentSchema) { - '\/deploymentTemplate.json#$' { - Write-Verbose 'Handling resource group level deployment' -Verbose - if (-not (Get-AzResourceGroup -Name '${{ parameters.resourceGroupName }}' -ErrorAction SilentlyContinue)) { - Write-Verbose 'Deploying resource group [${{ parameters.resourceGroupName }}]' -Verbose - $rgInputObject = @{ - Name = '${{ parameters.resourceGroupName }}' - Location = '${{ parameters.location }}' - } - New-AzResourceGroup @rgInputObject - } - if ('${{ parameters.removeDeployment }}' -eq 'true') { - Write-Output "Because the subsequent removal is enabled after the Module ${{ parameters.moduleName }} has been deployed, the following tags (moduleName: ${{ parameters.moduleName }}) are now set on the resource." - Write-Output "This is necessary so that the later running Removal Stage can remove the corresponding Module from the Resource Group again." - $DeploymentInputs += @{ - Tags = @{ RemoveModule = "${{ parameters.moduleName }}"; } - } - } - New-AzResourceGroupDeployment @DeploymentInputs -ResourceGroupName '${{ parameters.resourceGroupName }}' - break - } - '\/subscriptionDeploymentTemplate.json#$' { - Write-Verbose 'Handling subscription level deployment' -Verbose - $DeploymentInputs += @{ - Location = '${{ parameters.location }}' - } - New-AzSubscriptionDeployment @DeploymentInputs - break - } - '\/managementGroupDeploymentTemplate.json#$' { - Write-Verbose 'Handling management group level deployment' -Verbose - $DeploymentInputs += @{ - ManagementGroupId = '${{ parameters.managementGroupId }}' - Location = '${{ parameters.location }}' - } - New-AzManagementGroupDeployment @DeploymentInputs - break - } - '\/tenantDeploymentTemplate.json#$' { - Write-Verbose 'Handling tenant level deployment' -Verbose - $DeploymentInputs += @{ - Location = '${{ parameters.location }}' - } - New-AzTenantDeployment @DeploymentInputs - break - } - default { - throw "[$deploymentSchema] is a non-supported ARM template schema" - } - } - enabled: ${{ parameters.enabled }} diff --git a/_in progress/.pipelines/.templates/pipeline.jobs.script.yml b/_in progress/.pipelines/.templates/pipeline.jobs.script.yml deleted file mode 100644 index b358ddc..0000000 --- a/_in progress/.pipelines/.templates/pipeline.jobs.script.yml +++ /dev/null @@ -1,58 +0,0 @@ -parameters: - jobName: - moduleName: - scriptType: - scriptFilePath: - dependsOn: [] - environment: '' - timeoutInMinutes: 90 - artifactFeedPath: '$(artifactFeedPath)' - serviceConnection: '$(serviceConnection)' - vmImage: $(vmImage) - poolName: $(poolName) - location: '$(location)' - resourceGroupName: '$(resourceGroupName)' - managementGroupId: '$(managementGroupId)' - displayName: 'Deploy module' - whatif: false - enabled: true - -jobs: - - deployment: ${{ parameters.jobName }}${{ parameters.whatif }} - ${{ if eq( parameters.whatif, true) }}: - displayName: ${{ parameters.displayName }} WhatIf - ${{ if ne( parameters.whatif, true) }}: - displayName: ${{ parameters.displayName }} - ${{ if ne( parameters.dependsOn, '') }}: - dependsOn: - - ${{ each dependency in parameters.dependsOn }}: - - ${{ dependency }}${{ parameters.whatif }} - environment: ${{ parameters.environment }} - timeoutInMinutes: ${{ parameters.timeoutInMinutes }} - pool: - ${{ if ne(parameters.vmImage, '') }}: - vmImage: ${{ parameters.vmImage }} - ${{ if ne(parameters.poolName, '') }}: - name: ${{ parameters.poolName }} - strategy: - runOnce: - deploy: - steps: - - checkout: self - persistCredentials: true - - task: AzureCLI@2 - displayName: 'Invoke Command via Azure CLI on [${{ parameters.vmName }}] in [${{ parameters.resourceGroupName }}]' - enabled: ${{ parameters.enabled }} - inputs: - azureSubscription: ${{ parameters.serviceConnection }} - scriptType: 'ps' - scriptLocation: inlineScript - inlineScript: | - if ('${{ parameters.scriptType }}' -eq 'bash') { - az vm run-command invoke -g ${{ parameters.resourceGroupName }} -n ${{ parameters.vmName }} --command-id RunShellScript \ - --scripts @${{ parameters.scriptFilePath }} --parameters "arg1=somefoo" "arg2=somebar" - } else { - az vm run-command invoke -g ${{ parameters.resourceGroupName }} -n ${{ parameters.vmName }} --command-id RunPowerShellScript \ - --scripts @${{ parameters.scriptFilePath }} --parameters "arg1=somefoo" "arg2=somebar" - } - Write-Output "$(Build.SourcesDirectory)/$(environmentPath)/" diff --git a/_in progress/Start-AzurePipelines.ps1 b/_in progress/Start-AzurePipelines.ps1 deleted file mode 100644 index da73d17..0000000 --- a/_in progress/Start-AzurePipelines.ps1 +++ /dev/null @@ -1,91 +0,0 @@ -<# -.SYNOPSIS -Start all specified DevOps pipelines in a target DevOps Project. - -.DESCRIPTION -Starts all specified DevOps pipelines in a target DevOps Project. -If this scripts is run within an Azure pipeline the environment variable AZURE_DEVOPS_EXT_PAT needs to be set with $(System.AccessToken) within your pipeline. -Since tty is not supported within a pipelune run, az devops login is using the token which is set via AZURE_DEVOPS_EXT_PAT. - -.REQUIREMENTS -- Azure CLI 2.13.0 -- Azure CLI extension devops 0.18.0 -- Repository for which the pipeline needs to be configured. -- The '' Build Service needs 'Edit build pipeline' permissions -Reference: https://docs.microsoft.com/en-us/azure/devops/pipelines/policies/permissions?view=azure-devops#pipeline-permissions - -.PARAMETER OrganizationName -Required. The name of the Azure DevOps organization. - -.PARAMETER ProjectName -Required. The name of the Azure DevOps project. - -.PARAMETER AzureDevOpsPAT -Required. The access token with appropriate permissions to create Azure Pipelines. -Usually the System.AccessToken from an Azure Pipeline instance run has sufficient permissions as well. -Reference: https://docs.microsoft.com/en-us/azure/devops/pipelines/process/access-tokens?view=azure-devops&tabs=yaml#how-do-i-determine-the-job-authorization-scope-of-my-yaml-pipeline -Needs at least the permissions: -- Agent Pool: Read -- Build: Read & execute -- Service Connections: Read & query - -.PARAMETER Branch -Optional. Name of the branch on which the pipeline run is to be queued. Default: refs/heads/main - -.PARAMETER FolderPath -Optional. Folder path of pipeline. Default is root '\' level folder. - -.EXAMPLE -$inputObject = @{ - OrganizationName = 'Contoso' - ProjectName = 'CICD' - AzureDevOpsPAT = '' -} -StartPipeline @inputObject - -#> -function Start-AzurePipelines { - - [CmdletBinding()] - param ( - [Parameter(Mandatory = $true)] - [string] $OrganizationName, - - [Parameter(Mandatory = $true)] - [string] $ProjectName, - - [Parameter(Mandatory = $true)] - [string] $AzureDevOpsPAT, - - [Parameter(Mandatory = $false)] - [string] $Branch = 'refs/heads/main', - - [Parameter(Mandatory = $false)] - [string] $FolderPath = '\' - ) - - Write-Verbose "Trying to login to Azure DevOps project $organizationName/$projectName with a PAT" - $orgUrl = "https://dev.azure.com/$organizationName/" - # $AzureDevOpsPAT | az devops login - - Write-Verbose "Set default Azure DevOps configuration to $organizationName and $projectName" - az devops configure --defaults organization=$orgUrl project=$projectName --use-git-aliases $true - - Write-Verbose "Get and list all Azure Pipelines in $folderPath" - $azurePipelines = az pipelines list --organization $orgUrl --project $projectName --folder-path $folderPath | ConvertFrom-Json | Sort-Object name - Write-Verbose ('Found [{0}] Azure Pipeline(s) in project [{1}]' -f $azurePipelines.Count, $projectName) - - Write-Verbose '----------------------------------' - foreach ($pipeline in $azurePipelines) { - Write-Verbose ('Start Azure pipeline [{0}]' -f $pipeline.name) - - $inputObject = @( - '--id', $pipeline.id, - '--branch', $branch, - '--folder-path', $pipeline.path, - '--name', $pipeline.name - ) - - $pipelineresult = az pipelines run @inputObject - } -} \ No newline at end of file diff --git a/_in progress/Stop-AzurePipelines.ps1 b/_in progress/Stop-AzurePipelines.ps1 deleted file mode 100644 index 3d03ddc..0000000 --- a/_in progress/Stop-AzurePipelines.ps1 +++ /dev/null @@ -1,20 +0,0 @@ -# Replace the values in this for the variables with your own. -$AzureDevOpsPAT = "{Personal Access Token}" -$OrganizationName = "{DevOps Organization Name}" -$ProjectName = "{DevOps Project Name}" - -$AzureDevOpsAuthenicationHeader = @{Authorization = 'Basic ' + [Convert]::ToBase64String([Text.Encoding]::ASCII.GetBytes(":$($AzureDevOpsPAT)")) } - -$Uri= "https://dev.azure.com/$($OrganizationName)/$($ProjectName)/_apis/build/builds?statusFilter=notStarted&api-version=5.1" - -$PendingJobs = Invoke-RestMethod -Uri $Uri -Headers $AzureDevOpsAuthenicationHeader -Method get -ContentType "application/json" -$JobsToCancel = $PendingJobs.value - -ForEach($build in $JobsToCancel) -{ - $build.status = "Cancelling" - $body = $build | ConvertTo-Json -Depth 10 - $urlToCancel = "https://dev.azure.com/$($OrganizationName)/$($ProjectName)/_apis/build/builds/$($build.id)?api-version=5.1" - $urlToCancel - # Invoke-RestMethod -Uri $urlToCancel -Method Patch -ContentType application/json -Body $body -Header $AzureDevOpsAuthenicationHeader -} \ No newline at end of file diff --git a/_in progress/clone-azure-avm-repos.ps1 b/_in progress/clone-azure-avm-repos.ps1 deleted file mode 100755 index 4047b6a..0000000 --- a/_in progress/clone-azure-avm-repos.ps1 +++ /dev/null @@ -1,133 +0,0 @@ -#!/usr/bin/env pwsh - -# Script to clone or update all terraform-azurerm-avm-* repos from Azure GitHub organization -# Date: June 20, 2025 - -$ErrorActionPreference = "Stop" -$orgName = "Azure" -$repoPrefix = "terraform-azurerm-avm-" -$baseDirectory = "/Users/segraef/Git/GitHub/$orgName" - -# Function to check if user is authenticated to GitHub -function Test-GithubAuth { - try { - $authStatus = gh auth status 2>&1 - if ($authStatus -match "Logged in to github.com") { - Write-Host "✅ Already authenticated to GitHub" -ForegroundColor Green - return $true - } else { - Write-Host "⚠️ Not authenticated to GitHub" -ForegroundColor Yellow - return $false - } - } - catch { - Write-Host "⚠️ Not authenticated to GitHub" -ForegroundColor Yellow - return $false - } -} - -# Function to authenticate to GitHub -function Connect-Github { - Write-Host "🔑 Please authenticate to GitHub..." - gh auth login - if (-not (Test-GithubAuth)) { - Write-Host "❌ Failed to authenticate to GitHub. Exiting script." -ForegroundColor Red - exit 1 - } -} - -# Function to update existing repository -function Update-Repository { - param ( - [string]$repoPath, - [string]$repoName - ) - - Write-Host "🔄 Updating repository: $repoName" -ForegroundColor Cyan - Set-Location $repoPath - - # Check if we're on main branch, if not switch to it - $currentBranch = git branch --show-current - if ($currentBranch -ne "main") { - Write-Host " Switching to main branch..." - git switch main - } - - # Fetch and pull latest changes - Write-Host " Fetching latest changes..." - git fetch --all - Write-Host " Pulling latest changes..." - git pull - Write-Host " ✅ Repository updated: $repoName" -ForegroundColor Green -} - -# Function to clone new repository -function New-Repository { - param ( - [string]$repoPath, - [string]$repoName, - [string]$repoUrl - ) - - Write-Host "📥 Cloning repository: $repoName" -ForegroundColor Magenta - git clone $repoUrl $repoPath - if ($LASTEXITCODE -eq 0) { - Write-Host " ✅ Repository cloned: $repoName" -ForegroundColor Green - } else { - Write-Host " ❌ Failed to clone repository: $repoName" -ForegroundColor Red - } -} - -# Check if authenticated to GitHub -if (-not (Test-GithubAuth)) { - Connect-Github -} - -# Create base directory if it doesn't exist -if (-not (Test-Path $baseDirectory)) { - Write-Host "📁 Creating base directory: $baseDirectory" -ForegroundColor Yellow - New-Item -ItemType Directory -Path $baseDirectory -Force | Out-Null -} - -# Change to base directory -Set-Location $baseDirectory - -# Get all repositories starting with the prefix -Write-Host "🔍 Searching for repositories with prefix: $repoPrefix in $orgName organization..." -ForegroundColor Blue -$repos = gh repo list $orgName --json name,url --limit 1000 | ConvertFrom-Json | Where-Object { $_.name -like "$repoPrefix*" } - -if (-not $repos) { - Write-Host "❌ No repositories found matching the prefix: $repoPrefix" -ForegroundColor Red - exit 1 -} - -Write-Host "🎉 Found $($repos.Count) repositories matching the prefix" -ForegroundColor Green - -# Process each repository -foreach ($repo in $repos) { - $repoName = $repo.name - $repoUrl = $repo.url - $repoPath = Join-Path $baseDirectory $repoName - - # Check if repository exists locally - if (Test-Path $repoPath) { - # Check if it's a Git repository - if (Test-Path (Join-Path $repoPath ".git")) { - # Update repository - Update-Repository -repoPath $repoPath -repoName $repoName - } else { - Write-Host "⚠️ Directory exists but is not a Git repository: $repoName" -ForegroundColor Yellow - # Rename existing directory - $backupPath = "$repoPath-backup-$(Get-Date -Format 'yyyyMMddHHmmss')" - Write-Host " Moving existing directory to $backupPath" - Move-Item -Path $repoPath -Destination $backupPath - # Clone repository - New-Repository -repoPath $repoPath -repoName $repoName -repoUrl $repoUrl - } - } else { - # Clone repository - New-Repository -repoPath $repoPath -repoName $repoName -repoUrl $repoUrl - } -} - -Write-Host "✅ All repositories processed successfully!" -ForegroundColor Green diff --git a/_in progress/gh-ratelimit b/_in progress/gh-ratelimit deleted file mode 100644 index a53132c..0000000 --- a/_in progress/gh-ratelimit +++ /dev/null @@ -1,184 +0,0 @@ -#!/usr/bin/env bash -# gh-ratelimit - GitHub API rate-limit monitor for Bash and POSIX-style shells. -# -# What it does: -# Queries GitHub's /rate_limit endpoint through GitHub CLI and renders the -# current API buckets sorted by pressure so the tightest buckets appear first. -# -# Requirements: -# - gh (GitHub CLI) -# - jq -# - an authenticated gh session -# -# Setup: -# macOS: brew install gh jq -# Ubuntu: sudo apt-get install gh jq -# Then authenticate once: -# gh auth login -# -# Notes: -# - This script uses gh for both API access and auth context. -# - Bash version requires jq for parsing and pretty-printing JSON. -# - GitHub's /rate_limit endpoint is exempt from rate limiting, so polling it is safe. -# -# Usage: -# gh-ratelimit -# gh-ratelimit -w -# gh-ratelimit -w -i 5 -# gh-ratelimit -j -# gh-ratelimit -q -# gh-ratelimit -h -# -# Options: -# -w, --watch Refresh continuously. -# -i, --interval Watch refresh interval in seconds. Default: 10. -# -j, --json Print the raw GitHub API response as formatted JSON. -# -q, --quiet Only show buckets that are not full and have a non-zero limit. -# -h, --help Show this help text. -# -# Examples: -# gh-ratelimit # one-time snapshot -# gh-ratelimit -q # only show buckets under pressure -# gh-ratelimit -w -i 60 # refresh every 60 seconds -# gh-ratelimit -j | jq . # inspect raw payload - -set -euo pipefail - -WATCH=0 -INTERVAL=10 -JSON=0 -QUIET=0 - -show_usage() { - awk 'NR == 1 { next } /^#/ { sub(/^# ?/, ""); print; next } { exit }' "$0" -} - -while [[ $# -gt 0 ]]; do - case "$1" in - -w|--watch) WATCH=1 ;; - -i|--interval) INTERVAL="${2:-10}"; shift ;; - -j|--json) JSON=1 ;; - -q|--quiet) QUIET=1 ;; - -h|--help) - show_usage - exit 0 - ;; - *) echo "unknown arg: $1" >&2; exit 2 ;; - esac - shift -done - -command -v gh >/dev/null || { echo "gh not installed" >&2; exit 1; } -command -v jq >/dev/null || { echo "jq not installed" >&2; exit 1; } - -if ! gh auth status >/dev/null 2>&1; then - echo "gh is installed but not authenticated. Run: gh auth login" >&2 - exit 1 -fi - -# ANSI colours (skip if not a TTY) -if [[ -t 1 ]]; then - C_RESET=$'\033[0m'; C_DIM=$'\033[2m'; C_BOLD=$'\033[1m' - C_RED=$'\033[31m'; C_YEL=$'\033[33m'; C_GRN=$'\033[32m'; C_CYN=$'\033[36m' -else - C_RESET=; C_DIM=; C_BOLD=; C_RED=; C_YEL=; C_GRN=; C_CYN= -fi - -bar() { - # bar - local rem="$1" lim="$2" width="${3:-24}" - if (( lim <= 0 )); then printf '%*s' "$width" ''; return; fi - local filled=$(( rem * width / lim )) - (( filled < 0 )) && filled=0 - (( filled > width )) && filled=$width - local empty=$(( width - filled )) - local colour="$C_GRN" - local pct=$(( rem * 100 / lim )) - if (( pct < 10 )); then colour="$C_RED" - elif (( pct < 33 )); then colour="$C_YEL" - fi - printf '%s' "$colour" - if (( filled > 0 )); then - printf '█%.0s' $(seq 1 "$filled") - fi - printf '%s' "$C_DIM" - if (( empty > 0 )); then - printf '░%.0s' $(seq 1 "$empty") - fi - printf '%s' "$C_RESET" -} - -human_reset() { - # Seconds-from-now → e.g. "in 31m12s" or "passed" - local target="$1" now diff m s - now=$(date +%s) - diff=$(( target - now )) - if (( diff <= 0 )); then echo "now"; return; fi - m=$(( diff / 60 )); s=$(( diff % 60 )) - if (( m > 0 )); then printf 'in %dm%02ds' "$m" "$s" - else printf 'in %ds' "$s" - fi -} - -render() { - local payload - if ! payload=$(gh api rate_limit 2>/dev/null); then - echo "${C_RED}gh api failed - are you authenticated? (gh auth status)${C_RESET}" >&2 - return 1 - fi - - if (( JSON )); then - echo "$payload" | jq . - return 0 - fi - - # Header - local user host now - user=$(gh api user --jq .login 2>/dev/null || echo '?') - host=$(gh auth status 2>&1 | awk -F'[ ()]+' '/Logged in to/{print $5; exit}' || echo 'github.com') - now=$(date '+%H:%M:%S') - printf '%s%sGitHub rate limits%s user=%s%s%s host=%s %s%s%s\n' \ - "$C_BOLD" "$C_CYN" "$C_RESET" "$C_BOLD" "$user" "$C_RESET" "$host" "$C_DIM" "$now" "$C_RESET" - echo - - # Sort: most-pressured (lowest %) first - local jq_filter=' - .resources - | to_entries - | map({ - key: .key, - rem: .value.remaining, - lim: .value.limit, - used: .value.used, - reset: .value.reset, - pct: (if .value.limit > 0 then (.value.remaining * 100 / .value.limit) else 100 end) - }) - | sort_by(.pct) - | .[] - | [.key, (.rem|tostring), (.lim|tostring), (.used|tostring), (.reset|tostring), ((.pct|floor)|tostring)] - | @tsv' - - local key rem lim used reset pct - while IFS=$'\t' read -r key rem lim used reset pct; do - if (( QUIET )) && (( rem == lim || lim == 0 )); then continue; fi - printf '%s%-26s%s %5d/%-5d %s %3d%% resets %s\n' \ - "$C_BOLD" "$key" "$C_RESET" \ - "$rem" "$lim" \ - "$(bar "$rem" "$lim" 24)" \ - "$pct" \ - "$(human_reset "$reset")" - done < <(echo "$payload" | jq -r "$jq_filter") -} - -if (( WATCH )); then - trap 'tput cnorm 2>/dev/null; exit 0' INT TERM - tput civis 2>/dev/null || true - while true; do - clear - render || true - printf '\n%srefresh every %ss - ctrl-c to exit%s\n' "$C_DIM" "$INTERVAL" "$C_RESET" - sleep "$INTERVAL" - done -else - render -fi diff --git a/_in progress/gh-ratelimit.ps1 b/_in progress/gh-ratelimit.ps1 deleted file mode 100644 index f120758..0000000 --- a/_in progress/gh-ratelimit.ps1 +++ /dev/null @@ -1,289 +0,0 @@ -#!/usr/bin/env pwsh - -<# -.SYNOPSIS -Displays GitHub API rate-limit buckets using GitHub CLI. - -.DESCRIPTION -Queries GitHub's /rate_limit endpoint through GitHub CLI and renders the -response as either formatted JSON or a terminal-friendly table sorted by the -most constrained buckets first. - -This PowerShell version only requires GitHub CLI. Unlike the Bash version, it -does not require jq because JSON parsing is handled natively by PowerShell. - -.REQUIREMENTS -- GitHub CLI: gh -- An authenticated gh session - -.SETUP -Install GitHub CLI, then authenticate once: - - gh auth login - -Examples: -- macOS: brew install gh -- Windows: winget install --id GitHub.cli -- Linux: see https://cli.github.com/ for distro-specific packages - -This script polls GitHub's /rate_limit endpoint, which is exempt from rate -limiting, so watch mode is safe to use. - -.PARAMETER Watch -Refresh continuously until interrupted. - -.PARAMETER Interval -Refresh interval in seconds for watch mode. Default is 10. - -.PARAMETER Json -Print the raw API response as formatted JSON. - -.PARAMETER Quiet -Only show buckets whose remaining value is below the limit and whose limit is -greater than zero. - -.PARAMETER Help -Show usage information. - -.EXAMPLE -pwsh -File ./gh-ratelimit.ps1 - -.EXAMPLE -pwsh -File ./gh-ratelimit.ps1 -Watch -Interval 60 - -.EXAMPLE -pwsh -File ./gh-ratelimit.ps1 -Quiet - -.EXAMPLE -pwsh -File ./gh-ratelimit.ps1 -Json -#> - -param( - [Alias('w')] - [switch]$Watch, - - [Alias('i')] - [ValidateRange(1, 86400)] - [int]$Interval = 10, - - [Alias('j')] - [switch]$Json, - - [Alias('q')] - [switch]$Quiet, - - [Alias('h')] - [switch]$Help -) - -Set-StrictMode -Version Latest -$ErrorActionPreference = 'Stop' - -function Show-Usage { - @' -gh-ratelimit - GitHub API rate-limit monitor - -Usage: - gh-ratelimit.ps1 # snapshot - gh-ratelimit.ps1 -Watch # watch mode (refresh every 10s) - gh-ratelimit.ps1 -Watch -Interval 5 - gh-ratelimit.ps1 -Json # raw JSON - gh-ratelimit.ps1 -Quiet # only buckets with limit > 0 and remaining < limit - gh-ratelimit.ps1 -Help - -Short flags: - -w -i 5 -j -q -h - -Requirements: - - gh (GitHub CLI) - - an authenticated gh session - -Setup: - 1. Install GitHub CLI. - macOS: brew install gh - Windows: winget install --id GitHub.cli - Linux: https://cli.github.com/ - 2. Authenticate once: - gh auth login - -Notes: - - This PowerShell version does not require jq. - - It uses gh for API calls and auth context. - - /rate_limit itself is exempt from rate limiting, so polling it is safe. - -Examples: - gh-ratelimit.ps1 - gh-ratelimit.ps1 -Quiet - gh-ratelimit.ps1 -Watch -Interval 60 - gh-ratelimit.ps1 -Json -'@ -} - -function Test-CommandExists { - param([Parameter(Mandatory = $true)][string]$Name) - - return $null -ne (Get-Command -Name $Name -ErrorAction SilentlyContinue) -} - -function Get-Style { - param([Parameter(Mandatory = $true)][string]$Code) - - if ([Console]::IsOutputRedirected -or $env:TERM -eq 'dumb') { - return '' - } - - return [char]27 + '[' + $Code + 'm' -} - -$C_RESET = Get-Style '0' -$C_DIM = Get-Style '2' -$C_BOLD = Get-Style '1' -$C_RED = Get-Style '31' -$C_YEL = Get-Style '33' -$C_GRN = Get-Style '32' -$C_CYN = Get-Style '36' - -function Get-Bar { - param( - [Parameter(Mandatory = $true)][int]$Remaining, - [Parameter(Mandatory = $true)][int]$Limit, - [int]$Width = 24 - ) - - if ($Limit -le 0) { - return ' ' * $Width - } - - $filled = [math]::Floor(($Remaining * $Width) / $Limit) - if ($filled -lt 0) { $filled = 0 } - if ($filled -gt $Width) { $filled = $Width } - $empty = $Width - $filled - $pct = [math]::Floor(($Remaining * 100) / $Limit) - - $colour = $C_GRN - if ($pct -lt 10) { - $colour = $C_RED - } elseif ($pct -lt 33) { - $colour = $C_YEL - } - - $filledText = if ($filled -gt 0) { '█' * $filled } else { '' } - $emptyText = if ($empty -gt 0) { '░' * $empty } else { '' } - - return "$colour$filledText$C_DIM$emptyText$C_RESET" -} - -function Get-HumanReset { - param([Parameter(Mandatory = $true)][long]$Target) - - $now = [DateTimeOffset]::UtcNow.ToUnixTimeSeconds() - $diff = $Target - $now - if ($diff -le 0) { - return 'now' - } - - $minutes = [math]::Floor($diff / 60) - $seconds = $diff % 60 - if ($minutes -gt 0) { - return ('in {0}m{1:00}s' -f $minutes, $seconds) - } - - return ('in {0}s' -f $seconds) -} - -function Get-HostName { - try { - $status = gh auth status 2>&1 | Out-String - $match = [regex]::Match($status, 'Logged in to\s+([^\s]+)') - if ($match.Success) { - return $match.Groups[1].Value - } - } catch { - } - - return 'github.com' -} - -function Render { - $payloadText = gh api rate_limit 2>$null - if (-not $payloadText) { - throw 'gh api failed - are you authenticated? (gh auth status)' - } - - if ($Json) { - $payloadText | ConvertFrom-Json | ConvertTo-Json -Depth 8 - return - } - - $payload = $payloadText | ConvertFrom-Json - - $user = '?' - try { - $user = (gh api user --jq .login 2>$null).Trim() - if (-not $user) { - $user = '?' - } - } catch { - } - - $hostName = Get-HostName - $now = Get-Date -Format 'HH:mm:ss' - Write-Output ("{0}{1}GitHub rate limits{2} user={3}{4}{5} host={6} {7}{8}{2}" -f $C_BOLD, $C_CYN, $C_RESET, $C_BOLD, $user, $C_RESET, $hostName, $C_DIM, $now) - Write-Output '' - - $entries = foreach ($property in $payload.resources.PSObject.Properties) { - $value = $property.Value - $pct = if ($value.limit -gt 0) { [math]::Floor(($value.remaining * 100) / $value.limit) } else { 100 } - [pscustomobject]@{ - Key = $property.Name - Remaining = [int]$value.remaining - Limit = [int]$value.limit - Used = [int]$value.used - Reset = [long]$value.reset - Pct = [int]$pct - } - } - - foreach ($entry in ($entries | Sort-Object Pct, Key)) { - if ($Quiet -and (($entry.Remaining -eq $entry.Limit) -or ($entry.Limit -eq 0))) { - continue - } - - $line = "{0}{1,-26}{2} {3,5}/{4,-5} {5} {6,3}% resets {7}" -f ` - $C_BOLD, $entry.Key, $C_RESET, $entry.Remaining, $entry.Limit, (Get-Bar -Remaining $entry.Remaining -Limit $entry.Limit -Width 24), $entry.Pct, (Get-HumanReset -Target $entry.Reset) - Write-Output $line - } -} - -if ($Help) { - Show-Usage - exit 0 -} - -if (-not (Test-CommandExists -Name 'gh')) { - Write-Error 'gh not installed' - exit 1 -} - -try { - gh auth status *> $null -} catch { - Write-Error 'gh is installed but not authenticated. Run: gh auth login' - exit 1 -} - -if ($Watch) { - while ($true) { - Clear-Host - try { - Render - } catch { - Write-Error $_.Exception.Message - } - Write-Output '' - Write-Output ("{0}refresh every {1}s - ctrl-c to exit{2}" -f $C_DIM, $Interval, $C_RESET) - Start-Sleep -Seconds $Interval - } -} else { - Render -} diff --git a/_in progress/prefix-dev-ae-devops-rg/keyVaults/parameters.json b/_in progress/prefix-dev-ae-devops-rg/keyVaults/parameters.json deleted file mode 100644 index e2cf3d4..0000000 --- a/_in progress/prefix-dev-ae-devops-rg/keyVaults/parameters.json +++ /dev/null @@ -1,53 +0,0 @@ -{ - "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentParameters.json#", - "contentVersion": "1.0.0.0", - "parameters": { - "name": { - "value": "prefix-dev-eus-kv-009" - }, - "softDeleteRetentionInDays": { - "value": 7 - }, - "enableRbacAuthorization": { - "value": false - }, - "secrets": { - "value": { - "secureList": [{ - "name": "exampleSecret", - "value": "secretValue", - "contentType": "", - "attributesExp": 1702648632, - "attributesNbf": 10000 - }] - } - }, - "accessPolicies": { - "value": [{ - "objectId": "<>", - "permissions": { - "keys": [ - "all" - ], - "secrets": [ - "all" - ] - }, - "tenantId": "<>" - }, - { - "objectId": "<>", - "permissions": { - "keys": [ - "all" - ], - "secrets": [ - "all" - ] - }, - "tenantId": "<>" - } - ] - } - } -} \ No newline at end of file diff --git a/_in progress/prefix-dev-ae-devops-rg/networkSecurityGroups/parameters.json b/_in progress/prefix-dev-ae-devops-rg/networkSecurityGroups/parameters.json deleted file mode 100644 index bdc5959..0000000 --- a/_in progress/prefix-dev-ae-devops-rg/networkSecurityGroups/parameters.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentParameters.json#", - "contentVersion": "1.0.0.0", - "parameters": { - "name": { - "value": "prefix-az-nsg-ado-001" - }, - "securityRules": { - "value": [{ - "name": "SSH", - "properties": { - "description": "SSH", - "protocol": "*", - "sourcePortRange": "*", - "destinationPortRange": "22", - "sourceAddressPrefix": "*", - "destinationAddressPrefix": "*", - "access": "Allow", - "priority": 100, - "direction": "Inbound" - } - }, { - "name": "RDP", - "properties": { - "description": "RDP", - "protocol": "*", - "sourcePortRange": "*", - "destinationPortRange": "3389", - "sourceAddressPrefix": "*", - "destinationAddressPrefix": "*", - "access": "Allow", - "priority": 110, - "direction": "Inbound" - } - }] - } - } -} \ No newline at end of file diff --git a/_in progress/prefix-dev-ae-devops-rg/parameters.json b/_in progress/prefix-dev-ae-devops-rg/parameters.json deleted file mode 100644 index 1f5fb39..0000000 --- a/_in progress/prefix-dev-ae-devops-rg/parameters.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentParameters.json#", - "contentVersion": "1.0.0.0", - "parameters": { - "name": { - "value": "prefix-dev-eus-devops-rg" - } - } -} \ No newline at end of file diff --git a/_in progress/prefix-dev-ae-devops-rg/pipeline.jobs.yml b/_in progress/prefix-dev-ae-devops-rg/pipeline.jobs.yml deleted file mode 100644 index 3a1b8c1..0000000 --- a/_in progress/prefix-dev-ae-devops-rg/pipeline.jobs.yml +++ /dev/null @@ -1,169 +0,0 @@ -# Artifacts need to be set in Platform\.pipelines\.templates\pipeline.artifacts.yml to be used for moduleName and moduleVersion. - -parameters: - vmImage: $(vmImage) - poolName: $(poolName) - serviceConnection: '$(serviceConnection)' - environment: '' - whatif: false - -jobs: -# Resource Group -# -------------- -- template: /.pipelines/.templates/pipeline.jobs.artifact.deploy.yml - parameters: - jobName: Deploy_ResourceGroup - displayName: 'Deploy ResourceGroup' - moduleName: '$(RGModuleName)' - moduleVersion: '$(RGModuleVersion)' - parameterFilePath: '${{ parameters.resourceGroupName }}/parameters.json' - vmImage: '${{ parameters.vmImage }}' - poolName: '${{ parameters.poolName }}' - serviceConnection: '${{ parameters.serviceConnection }}' - environment: '${{ parameters.environment }}' - whatif: '${{ parameters.whatif }}' - enabled: true - -# Storage Account -# ---------------- -- template: /.pipelines/.templates/pipeline.jobs.artifact.deploy.yml - parameters: - jobName: Deploy_Storage_Account - displayName: 'Deploy Storage Account' - moduleName: '$(StorageAccountsModuleName)' - moduleVersion: '$(StorageAccountsModuleVersion)' - parameterFilePath: '${{ parameters.resourceGroupName }}/storageAccounts/parameters.json' - vmImage: '${{ parameters.vmImage }}' - poolName: '${{ parameters.poolName }}' - serviceConnection: '${{ parameters.serviceConnection }}' - environment: '${{ parameters.environment }}' - whatif: '${{ parameters.whatif }}' - enabled: true - dependsOn: - - Deploy_ResourceGroup - -# Network Security Groups -# ---------------- -- template: /.pipelines/.templates/pipeline.jobs.artifact.deploy.yml - parameters: - jobName: Deploy_NSG - displayName: 'Deploy Network Security Groups' - moduleName: '$(NSGModuleName)' - moduleVersion: '$(NSGModuleVersion)' - parameterFilePath: '${{ parameters.resourceGroupName }}/networkSecurityGroups/parameters.json' - vmImage: '${{ parameters.vmImage }}' - poolName: '${{ parameters.poolName }}' - serviceConnection: '${{ parameters.serviceConnection }}' - environment: '${{ parameters.environment }}' - whatif: '${{ parameters.whatif }}' - enabled: true - dependsOn: - - Deploy_ResourceGroup - -# Key Vault -# ---------------- -- template: /.pipelines/.templates/pipeline.jobs.artifact.deploy.yml - parameters: - jobName: Deploy_Key_Vault - displayName: 'Deploy Key Vault' - moduleName: '$(KVModuleName)' - moduleVersion: '$(KVModuleVersion)' - parameterFilePath: '${{ parameters.resourceGroupName }}/keyVaults/parameters.json' - vmImage: '${{ parameters.vmImage }}' - poolName: '${{ parameters.poolName }}' - serviceConnection: '${{ parameters.serviceConnection }}' - environment: '${{ parameters.environment }}' - whatif: '${{ parameters.whatif }}' - enabled: true - dependsOn: - - Deploy_NSG - -# Virtual Machines (Linux) -# ---------------- -- template: /.pipelines/.templates/pipeline.jobs.artifact.deploy.yml - parameters: - jobName: Deploy_Virtual_Machines_Linux - displayName: 'Deploy Virtual Machines' - moduleName: '$(VMModuleName)' - moduleVersion: '0.4.1024-prerelease' - parameterFilePath: '${{ parameters.resourceGroupName }}/virtualMachines/vmadolin001.parameters.json' - vmImage: '${{ parameters.vmImage }}' - poolName: '${{ parameters.poolName }}' - serviceConnection: '${{ parameters.serviceConnection }}' - environment: '${{ parameters.environment }}' - whatif: '${{ parameters.whatif }}' - enabled: false - dependsOn: - - Deploy_Key_Vault - -# Virtual Machines (Windows) -# ---------------- -- template: /.pipelines/.templates/pipeline.jobs.artifact.deploy.yml - parameters: - jobName: Deploy_Virtual_Machines_Windows - displayName: 'Deploy Virtual Machines' - moduleName: '$(VMModuleName)' - moduleVersion: '0.4.1024-prerelease' - parameterFilePath: '${{ parameters.resourceGroupName }}/virtualMachines/vmadowin001.parameters.json' - vmImage: '${{ parameters.vmImage }}' - poolName: '${{ parameters.poolName }}' - serviceConnection: '${{ parameters.serviceConnection }}' - environment: '${{ parameters.environment }}' - whatif: '${{ parameters.whatif }}' - enabled: true - dependsOn: - - Deploy_Key_Vault - -# Virtual Machines (Windows) -# ---------------- -- template: /.pipelines/.templates/pipeline.jobs.artifact.deploy.yml - parameters: - jobName: Deploy_Virtual_Machines_Windows_2 - displayName: 'Deploy Virtual Machines' - moduleName: '$(VMModuleName)' - moduleVersion: '0.4.1024-prerelease' - parameterFilePath: '${{ parameters.resourceGroupName }}/virtualMachines/vmadowin002.parameters.json' - vmImage: '${{ parameters.vmImage }}' - poolName: '${{ parameters.poolName }}' - serviceConnection: '${{ parameters.serviceConnection }}' - environment: '${{ parameters.environment }}' - whatif: '${{ parameters.whatif }}' - enabled: true - dependsOn: - - Deploy_Key_Vault - -# Invoke Command -# ---------------- -- template: /.pipelines/.templates/pipeline.jobs.script.yml - parameters: - jobName: InvokeCommand - displayName: 'Invoke Command' - scriptType: 'Bash' # PowerShell - scriptFilePath: '${{ parameters.resourceGroupName }}/scripts/script.sh' - vmImage: '${{ parameters.vmImage }}' - poolName: '${{ parameters.poolName }}' - serviceConnection: '${{ parameters.serviceConnection }}' - environment: '${{ parameters.environment }}' - whatif: '${{ parameters.whatif }}' - enabled: false - dependsOn: - - Deploy_Key_Vault - -# Deploy Private Endpoint -# ---------------- -- template: /.pipelines/.templates/pipeline.jobs.artifact.deploy.yml - parameters: - jobName: Deploy_PE - displayName: 'Deploy Private Endpoints' - moduleName: '$(PEModuleName)' - moduleVersion: '$(PEModuleVersion)' - parameterFilePath: '${{ parameters.resourceGroupName }}/privateEndpoints/parameters.json' - vmImage: '${{ parameters.vmImage }}' - poolName: '${{ parameters.poolName }}' - serviceConnection: '${{ parameters.serviceConnection }}' - environment: '${{ parameters.environment }}' - whatif: '${{ parameters.whatif }}' - enabled: true - dependsOn: - - Deploy_Virtual_Machines_Windows - - Deploy_Virtual_Machines_Linux \ No newline at end of file diff --git a/_in progress/prefix-dev-ae-devops-rg/pipeline.variables.yml b/_in progress/prefix-dev-ae-devops-rg/pipeline.variables.yml deleted file mode 100644 index 0c9f883..0000000 --- a/_in progress/prefix-dev-ae-devops-rg/pipeline.variables.yml +++ /dev/null @@ -1,7 +0,0 @@ -variables: - resourceGroupName: 'prefix-dev-eus-devops-rg' - environmentPath: 'root/prefix-dev-eus-water-02 (<>)' - location: 'East US' - vmImage: 'ubuntu-latest' - poolName: 'prefix-NonProd-WindowsPool-001' # prefix-NonProd-WindowsPool-001 - serviceConnection: 'prefix-dev-water-sp' diff --git a/_in progress/prefix-dev-ae-devops-rg/pipeline.yml b/_in progress/prefix-dev-ae-devops-rg/pipeline.yml deleted file mode 100644 index ab0c833..0000000 --- a/_in progress/prefix-dev-ae-devops-rg/pipeline.yml +++ /dev/null @@ -1,26 +0,0 @@ -name: $(resourceGroupName) - -trigger: - branches: - include: - - main - paths: - include: - - root/prefix-dev-eus-water-02 (<>)/prefix-dev-eus-devops-rg/* - -variables: - - template: pipeline.variables.yml - - template: /.pipelines/.templates/pipeline.artifacts.yml - -stages: - - stage: Deployment - jobs: - - template: ./pipeline.jobs.yml - parameters: - vmImage: '$(vmImage)' - poolName: '$(poolName)' - serviceConnection: '$(serviceConnection)' - resourceGroupName: '$(resourceGroupName)' - parameterFilePath: '' - environment: DEV - whatif: false diff --git a/_in progress/prefix-dev-ae-devops-rg/privateEndpoints/parameters.json b/_in progress/prefix-dev-ae-devops-rg/privateEndpoints/parameters.json deleted file mode 100644 index 6b25546..0000000 --- a/_in progress/prefix-dev-ae-devops-rg/privateEndpoints/parameters.json +++ /dev/null @@ -1,27 +0,0 @@ -{ - "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentParameters.json#", - "contentVersion": "1.0.0.0", - "parameters": { - "name": { - "value": "pe-dev-kvlt-001" - }, - "targetSubnetResourceId": { - "value": "/subscriptions/<>/resourceGroups/prefix-dev-eus-network-rg/providers/Microsoft.Network/virtualNetworks/prefix-dev-eus-spoke-vnet01/subnets/Privated-Endpoints" - }, - "serviceResourceId": { - "value": "/subscriptions/<>/resourceGroups/prefix-dev-eus-devops-rg/providers/Microsoft.KeyVault/vaults/prefix-dev-eus-kv-009" - }, - "groupId": { - "value": [ - "vault" - ] - }, - "privateDnsZoneGroups": { - "value": [{ - "privateDNSResourceIds": [ - "/subscriptions/<>/resourceGroups/prefix-id-eus-network-rg/providers/Microsoft.Network/privateDnsZones/privatelink.vaultcore.azure.net" - ] - }] - } - } -} \ No newline at end of file diff --git a/_in progress/prefix-dev-ae-devops-rg/scripts/Install-PipelineAgent.sh b/_in progress/prefix-dev-ae-devops-rg/scripts/Install-PipelineAgent.sh deleted file mode 100644 index b398d2a..0000000 --- a/_in progress/prefix-dev-ae-devops-rg/scripts/Install-PipelineAgent.sh +++ /dev/null @@ -1,93 +0,0 @@ -#!/bin/bash -set -e - -if [ -z "$AZP_URL" ]; then - echo 1>&2 "error: missing AZP_URL environment variable" - exit 1 -fi - -if [ -z "$AZP_TOKEN_FILE" ]; then - if [ -z "$AZP_TOKEN" ]; then - echo 1>&2 "error: missing AZP_TOKEN environment variable" - exit 1 - fi - - AZP_TOKEN_FILE=/azp/.token - echo -n $AZP_TOKEN > "$AZP_TOKEN_FILE" -fi - -unset AZP_TOKEN - -if [ -n "$AZP_WORK" ]; then - mkdir -p "$AZP_WORK" -fi - -rm -rf /azp/agent -mkdir /azp/agent -cd /azp/agent - -export AGENT_ALLOW_RUNASROOT="1" - -cleanup() { - if [ -e config.sh ]; then - print_header "Cleanup. Removing Azure Pipelines agent..." - - ./config.sh remove --unattended \ - --auth PAT \ - --token $(cat "$AZP_TOKEN_FILE") - fi -} - -print_header() { - lightcyan='\033[1;36m' - nocolor='\033[0m' - echo -e "${lightcyan}$1${nocolor}" -} - -# Let the agent ignore the token env variables -export VSO_AGENT_IGNORE=AZP_TOKEN,AZP_TOKEN_FILE - -print_header "1. Determining matching Azure Pipelines agent..." - -AZP_AGENT_RESPONSE=$(curl -LsS \ - -u user:$(cat "$AZP_TOKEN_FILE") \ - -H 'Accept:application/json;api-version=3.0-preview' \ - "$AZP_URL/_apis/distributedtask/packages/agent?platform=linux-x64") - -if echo "$AZP_AGENT_RESPONSE" | jq . >/dev/null 2>&1; then - AZP_AGENTPACKAGE_URL=$(echo "$AZP_AGENT_RESPONSE" \ - | jq -r '.value | map([.version.major,.version.minor,.version.patch,.downloadUrl]) | sort | .[length-1] | .[3]') -fi - -if [ -z "$AZP_AGENTPACKAGE_URL" -o "$AZP_AGENTPACKAGE_URL" == "null" ]; then - echo 1>&2 "error: could not determine a matching Azure Pipelines agent - check that account '$AZP_URL' is correct and the token is valid for that account" - exit 1 -fi - -print_header "2. Downloading and installing Azure Pipelines agent..." - -curl -LsS $AZP_AGENTPACKAGE_URL | tar -xz & wait $! - -source ./env.sh - -trap 'cleanup; exit 130' INT -trap 'cleanup; exit 143' TERM - -print_header "3. Configuring Azure Pipelines agent..." - -./config.sh --unattended \ - --agent "${AZP_AGENT_NAME:-$(hostname)}" \ - --url "$AZP_URL" \ - --auth PAT \ - --token $(cat "$AZP_TOKEN_FILE") \ - --pool "${AZP_POOL:-Default}" \ - --work "${AZP_WORK:-_work}" \ - --replace \ - --acceptTeeEula & wait $! - -print_header "4. Running Azure Pipelines agent..." - -# `exec` the node runtime so it's aware of TERM and INT signals -# AgentService.js understands how to handle agent self-update and restart -exec ./externals/node/bin/node ./bin/AgentService.js interactive -#pwsh \ No newline at end of file diff --git a/_in progress/prefix-dev-ae-devops-rg/scripts/Install-PipelineAgents.ps1 b/_in progress/prefix-dev-ae-devops-rg/scripts/Install-PipelineAgents.ps1 deleted file mode 100644 index 3c236a7..0000000 --- a/_in progress/prefix-dev-ae-devops-rg/scripts/Install-PipelineAgents.ps1 +++ /dev/null @@ -1,68 +0,0 @@ -New-Item -ItemType Directory "$PWD\agent\" -Force -Set-Location "$PWD\agent\" - -if (-not (Test-Path Env:AZP_URL)) { - Write-Error "error: missing AZP_URL environment variable" - exit 1 -} - -if (-not (Test-Path Env:AZP_TOKEN_FILE)) { - if (-not (Test-Path Env:AZP_TOKEN)) { - Write-Error "error: missing AZP_TOKEN environment variable" - exit 1 - } - - $Env:AZP_TOKEN_FILE = "$PWD\.token" - $Env:AZP_TOKEN | Out-File -FilePath $Env:AZP_TOKEN_FILE -} - -Remove-Item Env:AZP_TOKEN - -if (Test-Path Env:AZP_WORK) { - New-Item $Env:AZP_WORK -ItemType directory | Out-Null -} - -# Let the agent ignore the token env variables -$Env:VSO_AGENT_IGNORE = "AZP_TOKEN,AZP_TOKEN_FILE" - -Write-Host "1. Determining matching Azure Pipelines agent..." -ForegroundColor Cyan - -$base64AuthInfo = [Convert]::ToBase64String([Text.Encoding]::ASCII.GetBytes(":$(Get-Content ${Env:AZP_TOKEN_FILE})")) -$package = Invoke-RestMethod -Headers @{Authorization=("Basic $base64AuthInfo")} "$(${Env:AZP_URL})/_apis/distributedtask/packages/agent?platform=win-x64&`$top=1" -$packageUrl = $package[0].Value.downloadUrl - -Write-Host $packageUrl - -Write-Host "2. Downloading and installing Azure Pipelines agent $env:computername ..." -ForegroundColor Cyan - -$wc = New-Object System.Net.WebClient -$wc.DownloadFile($packageUrl, "$(Get-Location)\agent.zip") - -Expand-Archive -Path "agent.zip" -DestinationPath ".\" - -try -{ - if (-not (Test-Path Env:AZP_CLEANUP)) { - Write-Host "4. Configuring Azure Pipelines agent $env:computername ..." -ForegroundColor Cyan - - .\config.cmd --unattended ` - --agent "$(if (Test-Path Env:AZP_AGENT_NAME) { ${Env:AZP_AGENT_NAME} } else { ${Env:computername} })" ` - --url "$(${Env:AZP_URL})" ` - --auth PAT ` - --token "$(Get-Content ${Env:AZP_TOKEN_FILE})" ` - --pool "$(if (Test-Path Env:AZP_POOL) { ${Env:AZP_POOL} } else { 'Default' })" ` - --work "$(if (Test-Path Env:AZP_WORK) { ${Env:AZP_WORK} } else { '_work' })" ` - --replace ` - --runAsService ` - --windowsLogonAccount "NT AUTHORITY\SYSTEM" ` - --acceptTeeEula - } else { - Write-Host "Cleanup. Removing Azure Pipelines agent $env:computername ..." -ForegroundColor Cyan - - .\config.cmd remove --unattended ` - --auth PAT ` - --token "$(Get-Content ${Env:AZP_TOKEN_FILE})" - Remove-Item Env:AZP_CLEANUP - } -} -Catch {} \ No newline at end of file diff --git a/_in progress/prefix-dev-ae-devops-rg/scripts/devops_runtime_baremetal.sh b/_in progress/prefix-dev-ae-devops-rg/scripts/devops_runtime_baremetal.sh deleted file mode 100644 index 8492894..0000000 --- a/_in progress/prefix-dev-ae-devops-rg/scripts/devops_runtime_baremetal.sh +++ /dev/null @@ -1,146 +0,0 @@ -#!/bin/sh - -url=${1} -pat_token=${2} -agent_pool=${3} -agent_prefix=${4} -num_agent=${5} -admin_user=${6} -rover_version="${7}" - -error() { - local parent_lineno="$1" - local message="$2" - local code="${3:-1}" - if [[ -n "$message" ]] ; then - >&2 echo -e "\e[41mError on or near line ${parent_lineno}: ${message}; exiting with status ${code}\e[0m" - else - >&2 echo -e "\e[41mError on or near line ${parent_lineno}; exiting with status ${code}\e[0m" - fi - echo "" - exit "${code}" -} - -function cleanup { - echo "calling cleanup" - - echo "stopping the service" - sudo ./svc.sh stop || true - echo "uninstall the service" - sudo ./svc.sh uninstall || true - echo "un-register from AZDO" - sudo -u ${admin_user} ./config.sh remove --unattended --auth pat --token ${pat_token} || true -} - -set -ETe -trap 'error ${LINENO}' ERR 1 2 3 6 - -#strict mode, fail on error -# set -euo pipefail - -echo "start" - -echo "install Ubuntu packages" - -# To make it easier for build and release pipelines to run apt-get, -# configure apt to not require confirmation (assume the -y argument by default) -export DEBIAN_FRONTEND=noninteractive -echo "APT::Get::Assume-Yes \"true\";" > /etc/apt/apt.conf.d/90assumeyes - -apt-get update -apt-get install -y --no-install-recommends \ - ca-certificates \ - jq \ - apt-transport-https \ - docker.io \ - sudo - -echo "Allowing agent to run docker" - -usermod -aG docker ${admin_user} -systemctl daemon-reload -systemctl enable docker -service docker start -docker --version - -# Pull rover base image -echo "Rover docker image ${rover_version}" -docker pull "${rover_version}" 2>/dev/null - -echo "Installing Azure CLI" - -curl -sL https://aka.ms/InstallAzureCLIDeb | bash - -echo "install VSTS Agent" - -cd /home/${admin_user} -mkdir -p agent -cd agent - -echo "Installing required packages for agents" -### Install wget ### -apt-get install wget -y - -### Install Terraform ### -apt-get install zip unzip -y - -sudo wget -P /usr/src https://releases.hashicorp.com/terraform/0.13.5/terraform_0.13.5_linux_amd64.zip -unzip /usr/src/terraform_0.13.5_linux_amd64.zip -d /usr/src/terraform -sudo rm -rf /usr/src/terraform_0.13.5_linux_amd64.zip -sudo ln -s /usr/src/terraform/terraform /usr/bin/terraform - -# install packer -curl -fsSL https://apt.releases.hashicorp.com/gpg | sudo apt-key add - -sudo apt-add-repository "deb [arch=amd64] https://apt.releases.hashicorp.com $(lsb_release -cs) main" -sudo apt-get update && sudo apt-get install packer - -# Install powershell -sudo apt-get install -y apt-transport-https software-properties-common -wget -q https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb -sudo dpkg -i packages-microsoft-prod.deb -sudo apt-get update -sudo apt-get install -y powershell - -# install python3 & pip3 -sudo apt-get -y install python3 python3-pip - -# install ansible -sudo apt-get install -y ansible - -# install python netaddr -sudo apt-get install -y python-netaddr -sudo apt-get install -y python-dnspython - -# install git -sudo apt-get -y install git - -AGENTRELEASE="$(curl -s https://api.github.com/repos/Microsoft/azure-pipelines-agent/releases/latest | grep -oP '"tag_name": "v\K(.*)(?=")')" -AGENTURL="https://vstsagentpackage.azureedge.net/agent/${AGENTRELEASE}/vsts-agent-linux-x64-${AGENTRELEASE}.tar.gz" -echo "Release "${AGENTRELEASE}" appears to be latest" -echo "Downloading..." -curl -s ${AGENTURL} -o agent_package.tar.gz - -for agent_num in $(seq 1 ${num_agent}); do - agent_dir="agent-$agent_num" - mkdir -p "$agent_dir" - cd "$agent_dir" - echo "moving to $agent_dir" - - cleanup - - name="${agent_prefix}-${agent_num}" - echo "installing agent $name" - tar zxvf ../agent_package.tar.gz - chmod -R 777 . - echo "extracted" - ./bin/installdependencies.sh || true - echo "dependencies installed" - sudo -u ${admin_user} ./config.sh --unattended --url "${url}" --auth pat --token "${pat_token}" --pool "${agent_pool}" --agent "${name}" --acceptTeeEula --replace --work ./_work --runAsService - echo "configuration done" - ./svc.sh install - echo "service installed" - ./svc.sh start - echo "service started" - echo "config done" - cd .. -done diff --git a/_in progress/prefix-dev-ae-devops-rg/scripts/devops_runtime_baremetal_rhel.sh b/_in progress/prefix-dev-ae-devops-rg/scripts/devops_runtime_baremetal_rhel.sh deleted file mode 100644 index 3a0dd71..0000000 --- a/_in progress/prefix-dev-ae-devops-rg/scripts/devops_runtime_baremetal_rhel.sh +++ /dev/null @@ -1,158 +0,0 @@ -#!/bin/sh - -url=${1} -pat_token=${2} -agent_pool=${3} -agent_prefix=${4} -num_agent=${5} -admin_user=${6} -rover_version="${7}" - -error() { - local parent_lineno="$1" - local message="$2" - local code="${3:-1}" - if [[ -n "$message" ]] ; then - >&2 echo -e "\e[41mError on or near line ${parent_lineno}: ${message}; exiting with status ${code}\e[0m" - else - >&2 echo -e "\e[41mError on or near line ${parent_lineno}; exiting with status ${code}\e[0m" - fi - echo "" - exit "${code}" -} - -function cleanup { - echo "calling cleanup" - - echo "stopping the service" - sudo ./svc.sh stop || true - echo "uninstall the service" - sudo ./svc.sh uninstall || true - echo "un-register from AZDO" - sudo -u ${admin_user} ./config.sh remove --unattended --auth pat --token ${pat_token} || true -} - -set -ETe -trap 'error ${LINENO}' ERR 1 2 3 6 - -#strict mode, fail on error -# set -euo pipefail - -echo "start" - -echo "install Ubuntu packages" - - -yum update -y -# install docker -yum module remove container-tools -yum-config-manager \ - --add-repo \ - https://download.docker.com/linux/centos/docker-ce.repo -sed -i s/7/8/g /etc/yum.repos.d/docker-ce.repo -yum install docker-ce -y - -echo "Allowing agent to run docker" - -usermod -aG docker ${admin_user} -systemctl daemon-reload -systemctl enable docker -service docker start -docker --version - -# Pull rover base image - -# echo "Rover docker image ${rover_version}" -# docker pull "${rover_version}" 2>/dev/null - -echo "Installing Azure CLI" - -# install azure cli -sudo rpm --import https://packages.microsoft.com/keys/microsoft.asc -echo -e "[azure-cli] -name=Azure CLI -baseurl=https://packages.microsoft.com/yumrepos/azure-cli -enabled=1 -gpgcheck=1 -gpgkey=https://packages.microsoft.com/keys/microsoft.asc" | sudo tee /etc/yum.repos.d/azure-cli.repo -sudo yum install azure-cli -y - -echo "install VSTS Agent" - -cd /usr -mkdir -p agent -cd agent - -echo "Installing required packages for agents" - -### Install wget ### -yum install wget -y - -### Install Terraform ### -yum install zip unzip -y - -sudo wget -P /usr/src https://releases.hashicorp.com/terraform/0.13.5/terraform_0.13.5_linux_amd64.zip -unzip /usr/src/terraform_0.13.5_linux_amd64.zip -d /usr/src/terraform -sudo rm -rf /usr/src/terraform_0.13.5_linux_amd64.zip -sudo ln -s /usr/src/terraform/terraform /usr/bin/terraform - - -# install packer -curl -SL https://releases.hashicorp.com/packer/1.6.2/packer_1.6.2_linux_amd64.zip -o packer_1.6.2_linux_amd64.zip -unzip packer_1.6.2_linux_amd64.zip -sudo mv packer /usr/bin/packer - - -# Install powershell -curl https://packages.microsoft.com/config/rhel/7/prod.repo | sudo tee /etc/yum.repos.d/microsoft.repo -sudo yum install -y powershell - -# install python3 & pip3 -yum install -y python3 - -# install ansible -pip3 install ansible --user - -# install python netaddr -pip3 install netaddr -pip3 install dnspython -cp /root/.local/bin/* /usr/local/bin - -# install jq -JQ=/usr/bin/jq -curl https://stedolan.github.io/jq/download/linux64/jq > $JQ && chmod +x $JQ -ls -la $JQ - -# install git -sudo yum install git -y - -AGENTRELEASE="$(curl -s https://api.github.com/repos/Microsoft/azure-pipelines-agent/releases/latest | grep -oP '"tag_name": "v\K(.*)(?=")')" -AGENTURL="https://vstsagentpackage.azureedge.net/agent/${AGENTRELEASE}/vsts-agent-linux-x64-${AGENTRELEASE}.tar.gz" -echo "Release "${AGENTRELEASE}" appears to be latest" -echo "Downloading..." -curl -s ${AGENTURL} -o agent_package.tar.gz - -for agent_num in $(seq 1 ${num_agent}); do - agent_dir="agent-$agent_num" - mkdir -p "$agent_dir" - cd "$agent_dir" - echo "moving to $agent_dir" - - cleanup - - name="${agent_prefix}-${agent_num}" - echo "installing agent $name" - tar zxvf ../agent_package.tar.gz - chmod -R 777 . - echo "extracted" - ./bin/installdependencies.sh || true - echo "dependencies installed" - sudo -u ${admin_user} ./config.sh --unattended --url "${url}" --auth pat --token "${pat_token}" --pool "${agent_pool}" --agent "${name}" --acceptTeeEula --replace --work ./_work --runAsService - echo "configuration done" - ./svc.sh install - echo "service installed" - ./svc.sh start - echo "service started" - echo "config done" - cd .. -done diff --git a/_in progress/prefix-dev-ae-devops-rg/scripts/script.ps1 b/_in progress/prefix-dev-ae-devops-rg/scripts/script.ps1 deleted file mode 100644 index f789b68..0000000 --- a/_in progress/prefix-dev-ae-devops-rg/scripts/script.ps1 +++ /dev/null @@ -1,113 +0,0 @@ -# Agent Setup -## Set TLS 1.2 -[Net.ServicePointManager]::SecurityProtocol = "Tls, Tls11, Tls12, Ssl3" - -## Install Azure Pipelines Agent -$organization = "prefix" # Read-Host "Azure DevOps Organization Name" -$Env:AZP_URL = "https://dev.azure.com/$organization/" -$Env:AZP_TOKEN = "" # Read-Host "Azure DevOps Personal Access Token" -$Env:AZP_POOL= "prefix-NonProd-WindowsPool-001" # Read-Host "Azure DevOps Agent Pool " # -# $Env:AZP_CLEANUP = "true" - -if (-not (Test-Path "$PWD\agent\")) { - # Install Agent - New-Item -ItemType Directory "$PWD\agent\" -Force - Set-Location "$PWD\agent\" - - if (-not (Test-Path Env:AZP_URL)) { - Write-Error "error: missing AZP_URL environment variable" - exit 1 - } - - if (-not (Test-Path Env:AZP_TOKEN_FILE)) { - if (-not (Test-Path Env:AZP_TOKEN)) { - Write-Error "error: missing AZP_TOKEN environment variable" - exit 1 - } - - $Env:AZP_TOKEN_FILE = "$PWD\.token" - $Env:AZP_TOKEN | Out-File -FilePath $Env:AZP_TOKEN_FILE - } - - Remove-Item Env:AZP_TOKEN - - if (Test-Path Env:AZP_WORK) { - New-Item $Env:AZP_WORK -ItemType directory | Out-Null - } - - # Let the agent ignore the token env variables - $Env:VSO_AGENT_IGNORE = "AZP_TOKEN,AZP_TOKEN_FILE" - - Write-Host "1. Determining matching Azure Pipelines agent..." -ForegroundColor Cyan - - $base64AuthInfo = [Convert]::ToBase64String([Text.Encoding]::ASCII.GetBytes(":$(Get-Content ${Env:AZP_TOKEN_FILE})")) - $package = Invoke-RestMethod -Headers @{Authorization=("Basic $base64AuthInfo")} "$(${Env:AZP_URL})/_apis/distributedtask/packages/agent?platform=win-x64&`$top=1" - $packageUrl = $package[0].Value.downloadUrl - - Write-Host $packageUrl - - Write-Host "2. Downloading and installing Azure Pipelines agent $env:computername ..." -ForegroundColor Cyan - - $wc = New-Object System.Net.WebClient - $wc.DownloadFile($packageUrl, "$(Get-Location)\agent.zip") - - Expand-Archive -Path "agent.zip" -DestinationPath ".\" - - try - { - if (-not (Test-Path Env:AZP_CLEANUP)) { - Write-Host "4. Configuring Azure Pipelines agent $env:computername ..." -ForegroundColor Cyan - - .\config.cmd --unattended ` - --agent "$(if (Test-Path Env:AZP_AGENT_NAME) { ${Env:AZP_AGENT_NAME} } else { ${Env:computername} })" ` - --url "$(${Env:AZP_URL})" ` - --auth PAT ` - --token "$(Get-Content ${Env:AZP_TOKEN_FILE})" ` - --pool "$(if (Test-Path Env:AZP_POOL) { ${Env:AZP_POOL} } else { 'Default' })" ` - --work "$(if (Test-Path Env:AZP_WORK) { ${Env:AZP_WORK} } else { '_work' })" ` - --replace ` - --runAsService ` - --windowsLogonAccount "NT AUTHORITY\SYSTEM" ` - --acceptTeeEula - } else { - Write-Host "Cleanup. Removing Azure Pipelines agent $env:computername ..." -ForegroundColor Cyan - - .\config.cmd remove --unattended ` - --auth PAT ` - --token "$(Get-Content ${Env:AZP_TOKEN_FILE})" - Remove-Item Env:AZP_CLEANUP - } - } - Catch {} -} - -# Tools -Invoke-Expression ((New-Object System.Net.WebClient).DownloadString('https://community.chocolatey.org/install.ps1')) -choco install git -fy -# choco install azure-cli -choco install pwsh -fy - -## PowerShell 7 -# if (-not $Host.version.Major -lt 7) { -# Invoke-Expression "& { $(irm https://aka.ms/install-powershell.ps1) } -UseMSI" -# } - -## Azure CLI -$ProgressPreference = 'SilentlyContinue'; Invoke-WebRequest -Uri https://aka.ms/installazurecliwindows -OutFile .\AzureCLI.msi; Start-Process msiexec.exe -Wait -ArgumentList '/I AzureCLI.msi /quiet'; rm .\AzureCLI.msi - -## Azure Bicep (PowerShell) -$installPath = "C:\Program Files (x86)\Microsoft SDKs\Azure\CLI2\wbin" -(New-Object Net.WebClient).DownloadFile("https://github.com/Azure/bicep/releases/latest/download/bicep-win-x64.exe", "$installPath\bicep.exe") - -## Restart Service -Get-Service -Name vsts* | Stop-Service -Get-Service -Name vsts* | Start-Service - -## Az Module - -Install-Module -Name Az -AllowClobber -Scope AllUsers -Force -Install-Module -Name Pester -Force -SkipPublisherCheck - -Start-Process pwsh -ArgumentList "-Command 'Install-Module -Name Az -AllowClobber -Scope AllUsers -Force'" -Start-Process pwsh -ArgumentList "-Command 'az extension add --name azure-devops'" -Start-Process pwsh -ArgumentList "-Command 'az config set extension.use_dynamic_install=yes_without_prompt'" diff --git a/_in progress/prefix-dev-ae-devops-rg/scripts/script.sh b/_in progress/prefix-dev-ae-devops-rg/scripts/script.sh deleted file mode 100644 index a960af9..0000000 --- a/_in progress/prefix-dev-ae-devops-rg/scripts/script.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/sh - -# https://docs.microsoft.com/en-us/azure/devops/pipelines/agents/v2-linux?view=azure-devops - -# Creates directory & download ADO agent install files - -su - localAdminUser -c " -mkdir myagent && cd myagent -wget https://vstsagentpackage.azureedge.net/agent/2.186.1/vsts-agent-linux-x64-2.186.1.tar.gz -tar zxvf vsts-agent-linux-x64-2.186.1.tar.gz" - -# Unattended install - -su - localAdminUser -c " -./config.sh --unattended \ - --agent "$(hostname)" \ - --url "https://dev.azure.com/prefix" \ - --auth PAT \ - --token "PATPATPAT" \ - --pool "Default" \ - --replace \ - --acceptTeeEula & wait $!" - -cd /home/localAdminUser/ -#Configure as a service -sudo ./svc.sh install localAdminUser - -#Start svc -sudo ./svc.sh start \ No newline at end of file diff --git a/_in progress/prefix-dev-ae-devops-rg/storageAccounts/parameters.json b/_in progress/prefix-dev-ae-devops-rg/storageAccounts/parameters.json deleted file mode 100644 index 8af15f0..0000000 --- a/_in progress/prefix-dev-ae-devops-rg/storageAccounts/parameters.json +++ /dev/null @@ -1,44 +0,0 @@ -{ - "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentParameters.json#", - "contentVersion": "1.0.0.0", - "parameters": { - "name": { - "value": "<>" - }, - "storageAccountSku": { - "value": "Standard_ZRS" - }, - "allowBlobPublicAccess": { - "value": false - }, - "publicNetworkAccess": { - "value": "Disabled" - }, - "requireInfrastructureEncryption": { - "value": true - }, - "systemAssignedIdentity": { - "value": true - }, - "privateEndpoints": { - "value": [{ - "subnetResourceId": "/subscriptions/<>/resourceGroups/prefix-dev-eus-network-rg/providers/Microsoft.Network/virtualNetworks/prefix-dev-eus-spoke-vnet01/subnets/Privated-Endpoints", - "service": "blob" - }] - }, - "blobServices": { - "value": { - "containers": [{ - "name": "scripts", - "publicAccess": "None", - "roleAssignments": [{ - "roleDefinitionIdOrName": "Contributor", - "principalIds": [ - "7e8080a4-2827-46e2-9db6-8d365fb08747" - ] - }] - }] - } - } - } -} \ No newline at end of file diff --git a/_in progress/prefix-dev-ae-devops-rg/virtualMachines/vmadolin001.parameters.json b/_in progress/prefix-dev-ae-devops-rg/virtualMachines/vmadolin001.parameters.json deleted file mode 100644 index ed81ed2..0000000 --- a/_in progress/prefix-dev-ae-devops-rg/virtualMachines/vmadolin001.parameters.json +++ /dev/null @@ -1,73 +0,0 @@ -{ - "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentParameters.json#", - "contentVersion": "1.0.0.0", - "parameters": { - "name": { - "value": "vmadolin001" - }, - "osType": { - "value": "Linux" - }, - "encryptionAtHost": { - "value": false - }, - "imageReference": { - "value": { - "publisher": "Canonical", - "offer": "UbuntuServer", - "sku": "18.04-LTS", - "version": "latest" - } - }, - "osDisk": { - "value": { - "diskSizeGB": "128", - "managedDisk": { - "storageAccountType": "Premium_LRS" - } - } - }, - "vmSize": { - "value": "Standard_B12ms" - }, - "adminUsername": { - "value": "<>" - }, - "disablePasswordAuthentication": { - "value": true - }, - "publicKeys": { - "value": [{ - "path": "/home/<>/.ssh/authorized_keys", - "keyData": "ssh-rsa <>" - }] - }, - "nicConfigurations": { - "value": [{ - "nicSuffix": "-nic-01", - "ipConfigurations": [{ - "name": "ipconfig01", - "subnetId": "/subscriptions/<>/resourceGroups/prefix-dev-eus-network-rg/providers/Microsoft.Network/virtualNetworks/prefix-dev-eus-spoke-vnet01/subnets/ado-subnet", - "pipConfiguration": { - "publicIpNameSuffix": "-pip-01" - } - }], - "nsgId": "/subscriptions/<>/resourceGroups/prefix-dev-eus-devops-rg/providers/Microsoft.Network/networkSecurityGroups/prefix-az-nsg-ado-001" - }] - }, - "extensionCustomScriptConfig": { - "value": { - "enabled": false, - "fileData": [{ - "uri": "https://<>.blob.core.windows.net/scripts/devops_runtime_baremetal.sh", - "storageAccountId": "/subscriptions/<>/resourceGroups/prefix-dev-eus-devops-rg/providers/Microsoft.Storage/storageAccounts/<>" - }] - } - }, - "extensionCustomScriptProtectedSetting": { - "value": { - "commandToExecute": "devops_runtime_baremetal.sh" - } - } - } -} \ No newline at end of file diff --git a/_in progress/prefix-dev-ae-devops-rg/virtualMachines/vmadowin001.parameters.json b/_in progress/prefix-dev-ae-devops-rg/virtualMachines/vmadowin001.parameters.json deleted file mode 100644 index b162eb5..0000000 --- a/_in progress/prefix-dev-ae-devops-rg/virtualMachines/vmadowin001.parameters.json +++ /dev/null @@ -1,72 +0,0 @@ -{ - "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentParameters.json#", - "contentVersion": "1.0.0.0", - "parameters": { - "name": { - "value": "vmadowin001" - }, - "imageReference": { - "value": { - "publisher": "MicrosoftWindowsServer", - "offer": "WindowsServer", - "sku": "2016-Datacenter", - "version": "latest" - } - }, - "osType": { - "value": "Windows" - }, - "vmSize": { - "value": "Standard_B12ms" - }, - "encryptionAtHost": { - "value": false - }, - "osDisk": { - "value": { - "diskSizeGB": "128", - "managedDisk": { - "storageAccountType": "Premium_LRS" - } - } - }, - "adminUsername": { - "value": "<>" - }, - "adminPassword": { - "reference": { - "keyVault": { - "id": "/subscriptions/<>/resourceGroups/prefix-dev-eus-devops-rg/providers/Microsoft.KeyVault/vaults/prefix-dev-eus-kv-009" - }, - "secretName": "adminPassword" - } - }, - "nicConfigurations": { - "value": [{ - "nicSuffix": "-nic-01", - "ipConfigurations": [{ - "name": "ipconfig01", - "subnetId": "/subscriptions/<>/resourceGroups/prefix-dev-eus-network-rg/providers/Microsoft.Network/virtualNetworks/prefix-dev-eus-spoke-vnet01/subnets/ado-subnet", - "pipConfiguration": { - "publicIpNameSuffix": "-pip-01" - } - }], - "nsgId": "/subscriptions/<>/resourceGroups/prefix-dev-eus-devops-rg/providers/Microsoft.Network/networkSecurityGroups/prefix-az-nsg-ado-001" - }] - }, - "extensionCustomScriptConfig": { - "value": { - "enabled": false, - "fileData": [{ - "uri": "https://<>.blob.core.windows.net/scripts/script.ps1", - "storageAccountId": "/subscriptions/<>/resourceGroups/prefix-dev-eus-devops-rg/providers/Microsoft.Storage/storageAccounts/<>" - }] - } - }, - "extensionCustomScriptProtectedSetting": { - "value": { - "commandToExecute": "powershell -ExecutionPolicy Unrestricted -Command \"& .\\script.ps1\"" - } - } - } -} \ No newline at end of file diff --git a/_in progress/process-aiml-resource-groups.ps1 b/_in progress/process-aiml-resource-groups.ps1 deleted file mode 100755 index 4d5c4fd..0000000 --- a/_in progress/process-aiml-resource-groups.ps1 +++ /dev/null @@ -1,160 +0,0 @@ -#!/usr/bin/env pwsh - -# Script to iterate through Azure resource groups starting with "rg-aiml-" -# Date: June 20, 2025 - -$ErrorActionPreference = "Stop" -$rgPrefix = "rg-aiml-" - -# Role assignment parameters -$roleDefinitionName = "Owner" -$objectId = "" -$objectType = "Group" - -# Note: Azure CLI doesn't support --start-time and --end-time parameters directly -# Time-bound assignments must be done through the Azure Portal or PowerShell Az module - -# Function to check if user is authenticated to Azure -function Test-AzureAuth { - try { - $account = az account show --query "name" -o tsv 2>&1 - if ($LASTEXITCODE -eq 0) { - Write-Host "✅ Already authenticated to Azure as: $account" -ForegroundColor Green - return $true - } else { - Write-Host "⚠️ Not authenticated to Azure" -ForegroundColor Yellow - return $false - } - } - catch { - Write-Host "⚠️ Not authenticated to Azure" -ForegroundColor Yellow - return $false - } -} - -# Function to authenticate to Azure -function Connect-Azure { - Write-Host "🔑 Please authenticate to Azure..." - az login - if (-not (Test-AzureAuth)) { - Write-Host "❌ Failed to authenticate to Azure. Exiting script." -ForegroundColor Red - exit 1 - } - - # List available subscriptions and let user select one if there are multiple - $subscriptions = az account list --query "[].{Name:name, Id:id, IsDefault:isDefault}" -o json | ConvertFrom-Json - if ($subscriptions.Count -gt 1) { - Write-Host "Multiple subscriptions found. Please select one:" -ForegroundColor Yellow - for ($i = 0; $i -lt $subscriptions.Count; $i++) { - $defaultMark = if ($subscriptions[$i].IsDefault) { "[DEFAULT]" } else { "" } - Write-Host "[$i] $($subscriptions[$i].Name) ($($subscriptions[$i].Id)) $defaultMark" - } - - $selection = Read-Host "Enter the number of the subscription to use (press Enter for default)" - if ($selection -ne "") { - $selectedSubscription = $subscriptions[$selection].Id - Write-Host "Setting subscription to: $($subscriptions[$selection].Name)" - az account set --subscription $selectedSubscription - } - } -} - -# Check if authenticated to Azure -if (-not (Test-AzureAuth)) { - Connect-Azure -} - -# Get current subscription details -$currentSubscription = az account show --query "{Name:name, Id:id}" -o json | ConvertFrom-Json -Write-Host "Using subscription: $($currentSubscription.Name) ($($currentSubscription.Id))" -ForegroundColor Blue - -# Get all resource groups starting with the prefix -Write-Host "🔍 Searching for resource groups with prefix: $rgPrefix..." -ForegroundColor Blue -$resourceGroups = az group list --query "[?starts_with(name, '$rgPrefix')].{Name:name, Location:location, Tags:tags}" -o json | ConvertFrom-Json - -if (-not $resourceGroups -or $resourceGroups.Count -eq 0) { - Write-Host "❌ No resource groups found matching the prefix: $rgPrefix" -ForegroundColor Red - exit 1 -} - -Write-Host "🎉 Found $($resourceGroups.Count) resource groups matching the prefix" -ForegroundColor Green - -# Process each resource group -foreach ($rg in $resourceGroups) { - Write-Host "Processing resource group: $($rg.Name) in $($rg.Location)" -ForegroundColor Cyan - - # Get resources in the resource group - Write-Host " 📋 Listing resources in resource group $($rg.Name)..." - $resources = az resource list --resource-group $rg.Name --query "[].{Name:name, Type:type, Location:location}" -o json | ConvertFrom-Json - - Write-Host " 📊 Found $($resources.Count) resources in resource group $($rg.Name)" -ForegroundColor Yellow - - # Example: Display resources by type - $resourceTypes = $resources | Group-Object -Property Type - foreach ($type in $resourceTypes) { - Write-Host " 🔹 $($type.Count) resources of type: $($type.Name)" -ForegroundColor Magenta - foreach ($resource in $type.Group) { - Write-Host " - $($resource.Name) ($($resource.Location))" - } - } - - # Example: Get detailed information for specific resource types - # Uncomment and modify as needed - <# - $storageAccounts = $resources | Where-Object { $_.Type -eq "Microsoft.Storage/storageAccounts" } - if ($storageAccounts) { - Write-Host " 💾 Storage Account details:" -ForegroundColor Green - foreach ($sa in $storageAccounts) { - $saDetails = az storage account show --name $sa.Name --resource-group $rg.Name --query "{Name:name, Sku:sku.name, Kind:kind}" -o json | ConvertFrom-Json - Write-Host " - $($saDetails.Name) (SKU: $($saDetails.Sku), Kind: $($saDetails.Kind))" - } - } - #> - - # Example: Custom operations for each resource group - # This is where you can add your specific operations - Write-Host " 🔧 Performing custom operations for resource group $($rg.Name)..." -ForegroundColor Cyan - - # Assign the external group as Owner to the resource group - Write-Host " 👥 Assigning external group as Owner to resource group $($rg.Name)..." -ForegroundColor Yellow - - # Get the scope for the resource group - $scope = "/subscriptions/$($currentSubscription.Id)/resourceGroups/$($rg.Name)" - - # Check if role assignment already exists - $existingAssignment = az role assignment list --assignee $objectId --role $roleDefinitionName --scope $scope --query "[0]" -o json 2>$null | ConvertFrom-Json - - if ($existingAssignment) { - Write-Host " ⚠️ Role assignment already exists for this group on resource group $($rg.Name)" -ForegroundColor Yellow - } - else { - # Create the role assignment (no time-bound options in Azure CLI) - try { - $assignmentResult = az role assignment create ` - --role $roleDefinitionName ` - --assignee-object-id $objectId ` - --assignee-principal-type $objectType ` - --scope $scope -o json 2>&1 - - if ($LASTEXITCODE -eq 0) { - Write-Host " ✅ Successfully assigned role 'Owner' to external group for resource group $($rg.Name)" -ForegroundColor Green - } else { - Write-Host " ❌ Failed to assign role: $assignmentResult" -ForegroundColor Red - } - } - catch { - Write-Host " ❌ Error assigning role: $_" -ForegroundColor Red - } - } - - Write-Host " ✅ Completed processing resource group: $($rg.Name)" -ForegroundColor Green - Write-Host "" -} - -Write-Host "✅ All resource groups processed successfully!" -ForegroundColor Green -Write-Host "" -Write-Host "📝 Note: For time-bound role assignments (with start/end dates):" -ForegroundColor Yellow -Write-Host " Azure CLI doesn't support setting expiration dates directly." -ForegroundColor Yellow -Write-Host " To create time-bound assignments like shown in the screenshot, use PowerShell Az module instead:" -ForegroundColor Yellow -Write-Host " New-AzRoleAssignment -ObjectId $objectId -RoleDefinitionName $roleDefinitionName -Scope \$scope -ExpiryOn '2025-12-17T15:29:15Z'" -ForegroundColor Cyan -Write-Host "" diff --git a/docs/ado-builtin-variables.md b/docs/ado-builtin-variables.md new file mode 100644 index 0000000..3586898 --- /dev/null +++ b/docs/ado-builtin-variables.md @@ -0,0 +1,31 @@ +# Azure Pipelines built-in variables + +Reference notes for inspecting the built-in variables Azure Pipelines exposes to +a job. This is documentation, not a runnable PowerShell script. + +## Dump every variable from a job + +Run an inline Bash step that prints the environment, sorted: + +```bash +env | sort +``` + +## Equivalent inline step in `azure-pipelines.yml` + +The `steps` section is used inside a `job` section: + +```yaml +steps: # 'Steps' section is to be used inside 'job' section. + - task: Bash@3 + inputs: + targetType: 'inline' + script: 'env | sort' +``` + +## Reference + +Microsoft Learn: Define variables / built-in (predefined) variables for Azure +Pipelines. + + diff --git a/flask/server.py b/flask/server.py index 6bbc85b..4a8f5d0 100644 --- a/flask/server.py +++ b/flask/server.py @@ -1,4 +1,14 @@ -import os,requests,hashlib +"""Flask web front-end for Have I Been Pwned (HIBP) breach checks. + +Serves a small UI that lets a user check whether an email address appears in +known data breaches and whether a password appears in the Pwned Passwords +range API. The HIBP API key is read from the environment (via a .env file) +and is never exposed in any response. +""" +import os +import hashlib + +import requests from flask import Flask, render_template, request from dotenv import load_dotenv @@ -9,23 +19,33 @@ # Get the API key from the environment variables API_KEY = os.getenv('HIBP_API_KEY') +if not API_KEY: + raise RuntimeError('HIBP_API_KEY is not set. Add it to your environment or .env file.') + +# Request timeout (seconds) so a stalled call doesn't hang the web worker. +REQUEST_TIMEOUT = 10 + +# HIBP requires both an API key and a descriptive User-Agent. +HIBP_HEADERS = { + 'hibp-api-key': API_KEY, + 'User-Agent': 'segraef-Scripts-hibp-checker', +} + +# The Pwned Passwords range API is a separate, unauthenticated service: +# never send the HIBP API key to it. +PWD_HEADERS = {'User-Agent': 'segraef-Scripts-hibp-checker'} @app.route('/') def index(): return render_template('index.html') + @app.route('/cheat') def cheat(): return render_template('cheat.html') -@app.route('/test/') -def test(): - print(f'I got clicked and here is your API key: {API_KEY}') - return 'Click. Here is your API key: ' + API_KEY - - @app.route('/check', methods=['POST']) def check(): email = request.form.get('email') @@ -35,9 +55,10 @@ def check(): return "No email or password provided", 400 if email: # Send the email to the HIBP API - headers = {'hibp-api-key': API_KEY} response = requests.get( - f'https://haveibeenpwned.com/api/v3/breachedaccount/{email}', headers=headers) + f'https://haveibeenpwned.com/api/v3/breachedaccount/{email}', + headers=HIBP_HEADERS, + timeout=REQUEST_TIMEOUT) if response.status_code == 404: result["email"] = "Email not found in data breaches" elif response.status_code != 200: @@ -54,10 +75,11 @@ def check(): prefix = hashed_password[:5] suffix = hashed_password[5:] - # Send the hashed password to the HIBP API - headers = {'hibp-api-key': API_KEY} + # Send the hashed password to the Pwned Passwords API response = requests.get( - f'https://api.pwnedpasswords.com/range/{prefix}', headers=headers) + f'https://api.pwnedpasswords.com/range/{prefix}', + headers=PWD_HEADERS, + timeout=REQUEST_TIMEOUT) if response.status_code != 200: result["password"] = "Error checking password" else: @@ -73,4 +95,4 @@ def check(): if __name__ == '__main__': - app.run(port=5001, debug=True) + app.run(port=5001, debug=os.getenv('FLASK_DEBUG', 'false').lower() == 'true')