From f649bf717a71472f4ab08e08b7a35064f3339a59 Mon Sep 17 00:00:00 2001 From: Azure SDK Bot <53356347+azure-sdk@users.noreply.github.com> Date: Tue, 18 Aug 2020 12:04:46 -0700 Subject: [PATCH] Sync eng/common directory with azure-sdk-tools repository (#480) --- eng/common/README.md | 2 +- .../TestResources/New-TestResources.ps1.md | 43 +++++++++++++++---- eng/common/TestResources/README.md | 8 ++-- .../TestResources/Remove-TestResources.ps1.md | 32 +++++++++++--- .../TestResources/deploy-test-resources.yml | 4 +- .../TestResources/remove-test-resources.yml | 4 +- .../steps/replace-relative-links.yml | 4 +- .../templates/steps/verify-agent-os.yml | 2 +- .../templates/steps/verify-links.yml | 5 ++- eng/common/scripts/Verify-Links.ps1 | 41 ++++++++++++------ 10 files changed, 105 insertions(+), 40 deletions(-) diff --git a/eng/common/README.md b/eng/common/README.md index 7e9e197fc..f855e54ad 100644 --- a/eng/common/README.md +++ b/eng/common/README.md @@ -4,7 +4,7 @@ The `eng/common` directory contains engineering files that are common across the It should remain relatively small and only contain textual based files like scripts, configs, or templates. It should not contain binary files as they don't play well with git. -# Updating +## Updating Any updates to files in the `eng/common` directory should be made in the [azure-sdk-tools](https://github.com/azure/azure-sdk-tools) repo. All changes made will cause a PR to created in all subscribed azure-sdk language repos which will blindly replace all contents of diff --git a/eng/common/TestResources/New-TestResources.ps1.md b/eng/common/TestResources/New-TestResources.ps1.md index c9a462aae..e12f0a30d 100644 --- a/eng/common/TestResources/New-TestResources.ps1.md +++ b/eng/common/TestResources/New-TestResources.ps1.md @@ -8,12 +8,14 @@ schema: 2.0.0 # New-TestResources.ps1 ## SYNOPSIS + Deploys live test resources defined for a service directory to Azure. ## SYNTAX ### Default (Default) -``` + +```text New-TestResources.ps1 [-BaseName] -ServiceDirectory -TestApplicationId [-TestApplicationSecret ] [-TestApplicationOid ] [-DeleteAfterHours ] [-Location ] [-Environment ] [-AdditionalParameters ] [-CI] [-Force] [-WhatIf] @@ -21,7 +23,8 @@ New-TestResources.ps1 [-BaseName] -ServiceDirectory -TestAppli ``` ### Provisioner -``` + +```text New-TestResources.ps1 [-BaseName] -ServiceDirectory -TestApplicationId [-TestApplicationSecret ] [-TestApplicationOid ] -TenantId [-SubscriptionId ] -ProvisionerApplicationId -ProvisionerApplicationSecret [-DeleteAfterHours ] @@ -30,7 +33,8 @@ New-TestResources.ps1 [-BaseName] -ServiceDirectory -TestAppli ``` ## DESCRIPTION -Deploys live test resouces specified in test-resources.json files to a resource + +Deploys live test resources specified in test-resources.json files to a resource group. This script searches the directory specified in $ServiceDirectory recursively @@ -52,7 +56,8 @@ specified in $ProvisionerApplicationId and $ProvisionerApplicationSecret. ## EXAMPLES ### EXAMPLE 1 -``` + +```text Connect-AzAccount -Subscription "REPLACE_WITH_SUBSCRIPTION_ID" $testAadApp = New-AzADServicePrincipal -Role Owner -DisplayName 'azure-sdk-live-test-app' New-TestResources.ps1 ` @@ -69,7 +74,8 @@ Requires PowerShell 7 to use ConvertFrom-SecureString -AsPlainText or convert the SecureString to plaintext by another means. ### EXAMPLE 2 -``` + +```text New-TestResources.ps1 ` -BaseName 'Generated' ` -ServiceDirectory '$(ServiceDirectory)' ` @@ -84,7 +90,7 @@ New-TestResources.ps1 ` -Verbose ``` -Run this in an Azure DevOps CI (with approrpiate variables configured) before +Run this in an Azure DevOps CI (with appropriate variables configured) before executing live tests. The script will output variables as secrets (to enable log redaction). @@ -92,12 +98,13 @@ log redaction). ## PARAMETERS ### -BaseName + A name to use in the resource group and passed to the ARM template as 'baseName'. Limit $BaseName to enough characters to be under limit plus prefixes specified in the ARM template. See also https://docs.microsoft.com/azure/architecture/best-practices/resource-naming -Note: The value specified for this parameter will be overriden and generated +Note: The value specified for this parameter will be overridden and generated by New-TestResources.ps1 if $CI is specified. ```yaml @@ -113,6 +120,7 @@ Accept wildcard characters: False ``` ### -ServiceDirectory + A directory under 'sdk' in the repository root - optionally with subdirectories specified - in which to discover ARM templates named 'test-resources.json'. This can also be an absolute path or specify parent directories. @@ -130,6 +138,7 @@ Accept wildcard characters: False ``` ### -TestApplicationId + The AAD Application ID to authenticate the test runner against deployed resources. Passed to the ARM template as 'testApplicationId'. @@ -150,6 +159,7 @@ Accept wildcard characters: False ``` ### -TestApplicationSecret + Optional service principal secret (password) to authenticate the test runner against deployed resources. Passed to the ARM template as @@ -171,6 +181,7 @@ Accept wildcard characters: False ``` ### -TestApplicationOid + Service Principal Object ID of the AAD Test application. This is used to assign permissions to the AAD application so it can access tested features on the live @@ -195,6 +206,7 @@ Accept wildcard characters: False ``` ### -TenantId + The tenant ID of a service principal when a provisioner is specified. The same Tenant ID is used for Test Application and Provisioner Application. @@ -214,6 +226,7 @@ Accept wildcard characters: False ``` ### -SubscriptionId + Optional subscription ID to use for new resources when logging in as a provisioner. You can also use Set-AzContext if not provisioning. @@ -231,6 +244,7 @@ Accept wildcard characters: False ``` ### -ProvisionerApplicationId + The AAD Application ID used to provision test resources when a provisioner is specified. @@ -251,6 +265,7 @@ Accept wildcard characters: False ``` ### -ProvisionerApplicationSecret + A service principal secret (password) used to provision test resources when a provisioner is specified. @@ -271,6 +286,7 @@ Accept wildcard characters: False ``` ### -DeleteAfterHours + Optional. Positive integer number of hours from the current time to set the 'DeleteAfter' tag on the created resource group. @@ -283,7 +299,7 @@ created resource group. An optional cleanup process can delete resource groups whose "DeleteAfter" timestamp is less than the current time. -This isused for CI automation. +This is used for CI automation. ```yaml Type: Int32 @@ -298,6 +314,7 @@ Accept wildcard characters: False ``` ### -Location + Optional location where resources should be created. If left empty, the default is based on the cloud to which the template is being deployed: @@ -319,6 +336,7 @@ Accept wildcard characters: False ``` ### -Environment + Name of the cloud environment. The default is the Azure Public Cloud ('PublicCloud') @@ -336,6 +354,7 @@ Accept wildcard characters: False ``` ### -AdditionalParameters + Optional key-value pairs of parameters to pass to the ARM template(s). ```yaml @@ -351,6 +370,7 @@ Accept wildcard characters: False ``` ### -CI + Indicates the script is run as part of a Continuous Integration / Continuous Deployment (CI/CD) build (only Azure Pipelines is currently supported). @@ -367,6 +387,7 @@ Accept wildcard characters: False ``` ### -Force + Force creation of resources instead of being prompted. ```yaml @@ -382,6 +403,7 @@ Accept wildcard characters: False ``` ### -WhatIf + Shows what would happen if the cmdlet runs. The cmdlet is not run. @@ -398,6 +420,7 @@ Accept wildcard characters: False ``` ### -Confirm + Prompts you for confirmation before running the cmdlet. ```yaml @@ -413,7 +436,8 @@ Accept wildcard characters: False ``` ### -OutFile -save test environment settings into a test-resources.json.env file next to test-resources.json. + +save test environment settings into a test-resources.json.env file next to test-resources.json. The file is protected via DPAPI. The environment file would be scoped to the current repository directory. Note: Supported only on Windows. @@ -430,6 +454,7 @@ Accept wildcard characters: False ``` ### CommonParameters + This cmdlet supports the common parameters: -Debug, -ErrorAction, -ErrorVariable, -InformationAction, -InformationVariable, -OutVariable, -OutBuffer, -PipelineVariable, -Verbose, -WarningAction, and -WarningVariable. For more information, see [about_CommonParameters](http://go.microsoft.com/fwlink/?LinkID=113216). ## INPUTS diff --git a/eng/common/TestResources/README.md b/eng/common/TestResources/README.md index 3c0cd7c22..e02990d02 100644 --- a/eng/common/TestResources/README.md +++ b/eng/common/TestResources/README.md @@ -35,10 +35,10 @@ eng\common\TestResources\New-TestResources.ps1 ` -TestApplicationSecret (ConvertFrom-SecureString $sp.Secret -AsPlainText) ``` -If you are running this for a .NET project on Windows, the recommended method is to -add the `-OutFile` switch to the above command. This will save test environment settings -into a test-resources.json.env file next to test-resources.json. The file is protected via DPAPI. -The environment file would be scoped to the current repository directory and avoids the need to +If you are running this for a .NET project on Windows, the recommended method is to +add the `-OutFile` switch to the above command. This will save test environment settings +into a test-resources.json.env file next to test-resources.json. The file is protected via DPAPI. +The environment file would be scoped to the current repository directory and avoids the need to set environment variables or restart your IDE to recognize them. Along with some log messages, this will output environment variables based on diff --git a/eng/common/TestResources/Remove-TestResources.ps1.md b/eng/common/TestResources/Remove-TestResources.ps1.md index f9bc1803a..f8f1c001d 100644 --- a/eng/common/TestResources/Remove-TestResources.ps1.md +++ b/eng/common/TestResources/Remove-TestResources.ps1.md @@ -8,37 +8,43 @@ schema: 2.0.0 # Remove-TestResources.ps1 ## SYNOPSIS + Deletes the resource group deployed for a service directory from Azure. ## SYNTAX ### Default (Default) -``` + +```text Remove-TestResources.ps1 [-BaseName] [-Environment ] [-Force] [-WhatIf] [-Confirm] [] ``` ### Default+Provisioner -``` + +```text Remove-TestResources.ps1 [-BaseName] -TenantId [-SubscriptionId ] -ProvisionerApplicationId -ProvisionerApplicationSecret [-Environment ] [-Force] [-WhatIf] [-Confirm] [] ``` ### ResourceGroup+Provisioner -``` + +```text Remove-TestResources.ps1 -ResourceGroupName -TenantId [-SubscriptionId ] -ProvisionerApplicationId -ProvisionerApplicationSecret [-Environment ] [-Force] [-WhatIf] [-Confirm] [] ``` ### ResourceGroup -``` + +```text Remove-TestResources.ps1 -ResourceGroupName [-Environment ] [-Force] [-WhatIf] [-Confirm] [] ``` ## DESCRIPTION + Removes a resource group and all its resources previously deployed using New-TestResources.ps1. @@ -52,7 +58,8 @@ create resources. ## EXAMPLES ### EXAMPLE 1 -``` + +```text Remove-TestResources.ps1 -BaseName 'uuid123' -Force ``` @@ -60,7 +67,8 @@ Use the currently logged-in account to delete the resource group by the name of 'rg-uuid123' ### EXAMPLE 2 -``` + +```text Remove-TestResources.ps1 ` -ResourceGroupName "${env:AZURE_RESOURCEGROUP_NAME}" ` -TenantId '$(TenantId)' ` @@ -77,6 +85,7 @@ AZURE_RESOURCEGROUP_NAME. ## PARAMETERS ### -BaseName + A name to use in the resource group and passed to the ARM template as 'baseName'. This will delete the resource group named 'rg-\' @@ -93,6 +102,7 @@ Accept wildcard characters: False ``` ### -ResourceGroupName + The name of the resource group to delete. ```yaml @@ -108,6 +118,7 @@ Accept wildcard characters: False ``` ### -TenantId + The tenant ID of a service principal when a provisioner is specified. ```yaml @@ -123,6 +134,7 @@ Accept wildcard characters: False ``` ### -SubscriptionId + Optional subscription ID to use for new resources when logging in as a provisioner. You can also use Set-AzContext if not provisioning. @@ -140,6 +152,7 @@ Accept wildcard characters: False ``` ### -ProvisionerApplicationId + A service principal ID to provision test resources when a provisioner is specified. ```yaml @@ -155,6 +168,7 @@ Accept wildcard characters: False ``` ### -ProvisionerApplicationSecret + A service principal secret (password) to provision test resources when a provisioner is specified. ```yaml @@ -170,6 +184,7 @@ Accept wildcard characters: False ``` ### -ServiceDirectory + A directory under 'sdk' in the repository root - optionally with subdirectories specified - specified - in which to discover pre removal script named 'remove-test-resources-pre.json'. @@ -186,6 +201,7 @@ Accept wildcard characters: False ``` ### -Environment + Name of the cloud environment. The default is the Azure Public Cloud ('PublicCloud') @@ -203,6 +219,7 @@ Accept wildcard characters: False ``` ### -Force + Force removal of resource group without asking for user confirmation ```yaml @@ -218,6 +235,7 @@ Accept wildcard characters: False ``` ### -WhatIf + Shows what would happen if the cmdlet runs. The cmdlet is not run. @@ -234,6 +252,7 @@ Accept wildcard characters: False ``` ### -Confirm + Prompts you for confirmation before running the cmdlet. ```yaml @@ -249,6 +268,7 @@ Accept wildcard characters: False ``` ### CommonParameters + This cmdlet supports the common parameters: -Debug, -ErrorAction, -ErrorVariable, -InformationAction, -InformationVariable, -OutVariable, -OutBuffer, -PipelineVariable, -Verbose, -WarningAction, and -WarningVariable. For more information, see [about_CommonParameters](http://go.microsoft.com/fwlink/?LinkID=113216). ## INPUTS diff --git a/eng/common/TestResources/deploy-test-resources.yml b/eng/common/TestResources/deploy-test-resources.yml index b875a806b..87438443d 100644 --- a/eng/common/TestResources/deploy-test-resources.yml +++ b/eng/common/TestResources/deploy-test-resources.yml @@ -12,8 +12,8 @@ parameters: # "TenantId": "", # "TestApplicationId": "", # "TestApplicationSecret": "", -# "ProvisionerApplicationId": "", -# "ProvisionerApplicationSecret": "", +# "ProvisionerApplicationId": "", +# "ProvisionerApplicationSecret": "", # "Environment": "AzureCloud | AzureGov | AzureChina | " # } diff --git a/eng/common/TestResources/remove-test-resources.yml b/eng/common/TestResources/remove-test-resources.yml index 767f8c8c5..34beecd3c 100644 --- a/eng/common/TestResources/remove-test-resources.yml +++ b/eng/common/TestResources/remove-test-resources.yml @@ -12,8 +12,8 @@ parameters: # "TenantId": "", # "TestApplicationId": "", # "TestApplicationSecret": "", -# "ProvisionerApplicationId": "", -# "ProvisoinerApplicationSecret": "", +# "ProvisionerApplicationId": "", +# "ProvisionerApplicationSecret": "", # "Environment": "AzureCloud | AzureGov | AzureChina | " # } # The Remove-TestResources.ps1 script accommodates extra parameters so it will diff --git a/eng/common/pipelines/templates/steps/replace-relative-links.yml b/eng/common/pipelines/templates/steps/replace-relative-links.yml index 7ab2d76bc..fe90211c8 100644 --- a/eng/common/pipelines/templates/steps/replace-relative-links.yml +++ b/eng/common/pipelines/templates/steps/replace-relative-links.yml @@ -91,7 +91,7 @@ steps: else: return match.group(0) - def replace_prefined_relative_links(match, readme_location, root_folder, build_sha, repo_id): + def replace_predefined_relative_links(match, readme_location, root_folder, build_sha, repo_id): link_path = match.group(2).strip() if is_relative_link(link_path, readme_location): @@ -133,7 +133,7 @@ steps: content = re.sub( PREDEFINED_LINK_DISCOVERY_REGEX, - lambda match, readme_location=readme_location, root_folder=root_folder, build_sha=build_sha, repo_id=repo_id: replace_prefined_relative_links( + lambda match, readme_location=readme_location, root_folder=root_folder, build_sha=build_sha, repo_id=repo_id: replace_predefined_relative_links( match, readme_location, root_folder, build_sha, repo_id ), content, diff --git a/eng/common/pipelines/templates/steps/verify-agent-os.yml b/eng/common/pipelines/templates/steps/verify-agent-os.yml index b221583bc..4551279e1 100644 --- a/eng/common/pipelines/templates/steps/verify-agent-os.yml +++ b/eng/common/pipelines/templates/steps/verify-agent-os.yml @@ -9,7 +9,7 @@ steps: script: | # Script verifies the operating system for the platform on which it is being run # Used in build pipelines to verify the build agent os - # Variable: The friendly name or image name of the os to verfy against + # Variable: The friendly name or image name of the os to verify against from __future__ import print_function import sys import platform diff --git a/eng/common/pipelines/templates/steps/verify-links.yml b/eng/common/pipelines/templates/steps/verify-links.yml index a3d385bec..86d0013ac 100644 --- a/eng/common/pipelines/templates/steps/verify-links.yml +++ b/eng/common/pipelines/templates/steps/verify-links.yml @@ -11,4 +11,7 @@ steps: workingDirectory: $(Build.SourcesDirectory)/${{ parameters.Directory }} filePath: eng/common/scripts/Verify-Links.ps1 arguments: > - -urls $(dir -r -i *.md) -rootUrl "file://$(Build.SourcesDirectory)/${{ parameters.Directory }}" -recursive:$false -ignoreLinksFile ${{ parameters.IgnoreLinksFile }} + -urls $(dir -r -i *.md) + -rootUrl "file://$(Build.SourcesDirectory)/${{ parameters.Directory }}" + -recursive: $false + -ignoreLinksFile ${{ parameters.IgnoreLinksFile }} \ No newline at end of file diff --git a/eng/common/scripts/Verify-Links.ps1 b/eng/common/scripts/Verify-Links.ps1 index 5da025f9d..8ef7d518a 100644 --- a/eng/common/scripts/Verify-Links.ps1 +++ b/eng/common/scripts/Verify-Links.ps1 @@ -13,11 +13,13 @@ param ( [string] $rootUrl = "", # list of http status codes count as broken links. Defaults to 400, 401, 404, SocketError.HostNotFound = 11001, SocketError.NoData = 11004 [array] $errorStatusCodes = @(400, 401, 404, 11001, 11004), - # flag to allow resolving relative paths or not - [bool] $resolveRelativeLinks = $true + # flag to allow checking against azure sdk link guidance. + [bool] $checkLinkGuidance = $false ) $ProgressPreference = "SilentlyContinue"; # Disable invoke-webrequest progress dialog +# Regex of the locale keywords. +$locale = "/en-us/" function NormalizeUrl([string]$url){ if (Test-Path $url) { @@ -65,11 +67,13 @@ function ResolveUri ([System.Uri]$referralUri, [string]$link) } $linkUri = [System.Uri]$link; - if($resolveRelativeLinks){ + # Our link guidelines do not allow relative links so only resolve them when we are not + # validating links against our link guidelines (i.e. !$checkLinkGuideance) + if(!$checkLinkGuidance) { if (!$linkUri.IsAbsoluteUri) { # For rooted paths resolve from the baseUrl if ($link.StartsWith("/")) { - echo "rooturl = $rootUrl" + Write-Verbose "rooturl = $rootUrl" $linkUri = new-object System.Uri([System.Uri]$rootUrl, ".$link"); } else { @@ -114,13 +118,17 @@ function ParseLinks([string]$baseUri, [string]$htmlContent) function CheckLink ([System.Uri]$linkUri) { - if ($checkedLinks.ContainsKey($linkUri)) { return } + if ($checkedLinks.ContainsKey($linkUri)) { + return $checkedLinks[$linkUri] + } + + $linkValid = $true + Write-Verbose "Checking link $linkUri..." - Write-Verbose "Checking link $linkUri..." if ($linkUri.IsFile) { if (!(Test-Path $linkUri.LocalPath)) { LogWarning "Link to file does not exist $($linkUri.LocalPath)" - $script:badLinks += $linkUri + $linkValid = $false } } else { @@ -152,7 +160,7 @@ function CheckLink ([System.Uri]$linkUri) if ($statusCode -in $errorStatusCodes) { LogWarning "[$statusCode] broken link $linkUri" - $script:badLinks += $linkUri + $linkValid = $false } else { if ($null -ne $statusCode) { @@ -165,7 +173,14 @@ function CheckLink ([System.Uri]$linkUri) } } } - $checkedLinks[$linkUri] = $true; + + # Check if link uri includes locale info. + if ($checkLinkGuidance -and ($linkUri -match $locale)) { + LogWarning "DO NOT include locale $locale information in links: $linkUri." + $linkValid = $false + } + $checkedLinks[$linkUri] = $linkValid + return $linkValid } function GetLinks([System.Uri]$pageUri) @@ -218,7 +233,6 @@ if ($PSVersionTable.PSVersion.Major -lt 6) { LogWarning "Some web requests will not work in versions of PS earlier then 6. You are running version $($PSVersionTable.PSVersion)." } - $badLinks = @(); $ignoreLinks = @(); if (Test-Path $ignoreLinksFile) @@ -245,8 +259,11 @@ while ($pageUrisToCheck.Count -ne 0) Write-Host "Found $($linkUris.Count) links on page $pageUri"; foreach ($linkUri in $linkUris) { - CheckLink $linkUri - if ($recursive) { + $isLinkValid = CheckLink $linkUri + if (!$isLinkValid) { + $script:badLinks += $linkUri + } + if ($recursive -and $isLinkValid) { if ($linkUri.ToString().StartsWith($baseUrl) -and !$checkedPages.ContainsKey($linkUri)) { $pageUrisToCheck.Enqueue($linkUri); }