The pratical admin post kept VSC's keystore password encrypted, but with vSphere keystore passwords being so easily available on the Internet and NetApp's KB suggesting to place the password on the filesystem in plain-text, I did it the easy way and kept the password (netapp) in clear text in the config file. I've tested this script on both VSC 4.1 and 4.2 and it worked quite well.

You can copy and paste the code below, or download the script directly here.

Note: this script uses the Windows CA default WebServer Certificate Template. It also makes backups of your original certificates.

#########################################################################################
#
#   NetApp Virtual Storage Console SSL Generation and Replacement script version 0.5
#   Tested on VSC 4.1 and 4.2
#   No guarantees, warranties, etc.
#   Blog post: http://goo.gl/Cdlhb
#
#########################################################################################

# Place the certs on a network location if your farm is larger than one server
$basedir = "\\fileserver\share\Certs"

# Enter your Windows Certificate Authority information
# below. Make sure it responds to certutil requests.
$rootCA = "dc.base.local"
$rootCAName = "BASE-DC-CA"
$email = "vmware@base.local"
$org = "NetNerds"
$city = "Kaplan"
$state = "LA"
$country = "US"

# Enter the path of your VSC Installation
$vscdir = "C:\Program Files\NetApp\Virtual Storage Console"

# Enter the path of your openssl.exe (0.x and 1.x are supported).
# If you don't have OpenSSL already, the script will download it for you.
$openssldir = "C:\OpenSSL-Win32"
$openssl = $openssldir+"\bin\openssl.exe"

##############################################################################################
#
#	You probably don't need to change anything below.
#
##############################################################################################

$thisfqdn = ("$env:computername.$env:userdnsdomain").ToLower()
$backuptime = (Get-Date -uformat "%m%d%Y%H%M%S")
$wc = New-Object System.Net.WebClient

if (!(Test-Path "$basedir")) { $null = New-Item -Type Directory "$basedir" }

Write-Host -Foreground "DarkBlue" -Background "White" "Downloading root CA Cert.."
$url = "http://$rootCA/certsrv/certnew.cer?ReqID=CACert&Renewal=0&Enc=b64"
$root64 = "$basedir\Root64.cer"
$wc.UseDefaultCredentials = $true; $wc.DownloadFile($url,$root64)

if (!(Test-Path($openssl))) {
	Write-Host -Foreground "DarkBlue" -Background "White" "Downloading OpenSSL.."
	$null = New-Item -Type Directory $openssldir
	$sslurl = "https://openssl-for-windows.googlecode.com/files/openssl-0.9.8k_WIN32.zip"
	$sslzip = "$env:temp\openssl.zip"
	$wc.DownloadFile($sslurl,$sslzip)
	$env:path = $env:path + ";$openssldir"

	Write-Host -Foreground "DarkBlue" -Background "White" "Extracting OpenSSL.."
	$shellApplication = new-object -com shell.application
	$zipPackage = $shellApplication.NameSpace($sslzip)
	$destinationFolder = $shellApplication.NameSpace($openssldir)
	$destinationFolder.CopyHere($zipPackage.Items())
	Remove-Item $sslzip
} 

######################################################################
#
#	Generate Certs
#
######################################################################

Write-Host -Foreground "DarkBlue" -Background "White" "Generating service certs.."
	$service = "$thisfqdn-netapp"
	$server = $thisfqdn.Substring(0,$thisfqdn.IndexOf("."))

	$servicedir = "$basedir\$service"
	$servicecfg = "$servicedir\$service.cfg"
	$tempkey = "$servicedir\temp.key"
	$netappkey = "$servicedir\netapp.key"
	$netappcsr = "$servicedir\netapp.csr"
	$netappcrt = "$servicedir\netapp.crt"
	$netapppfx = "$servicedir\netapp.pfx"
	$chainpem = "$servicedir\chain.pem"
	$backupdir = "$servicedir\backup-$backuptime"
	$keyalias = "netapp" 

	if (Test-Path($servicedir)) { $null = Remove-Item "$servicedir\*.*" } else {$null = mkdir $servicedir } 

	Set-Content $servicecfg "[ req ]"
	Add-Content $servicecfg " default_md = sha512"
	Add-Content $servicecfg " default_bits = 2048"
	Add-Content $servicecfg " default_keyfile = netapp.key"
	Add-Content $servicecfg " distinguished_name = req_distinguished_name"
	Add-Content $servicecfg " encrypt_key = no"
	Add-Content $servicecfg " prompt = no"
	Add-Content $servicecfg " string_mask = nombstr"
	Add-Content $servicecfg " req_extensions = v3_req"
	Add-Content $servicecfg "`n[ v3_req ]"
	Add-Content $servicecfg " basicConstraints = CA:FALSE"
	Add-Content $servicecfg " keyUsage = digitalSignature, keyEncipherment, dataEncipherment"
	Add-Content $servicecfg " extendedKeyUsage = serverAuth"
	Add-Content $servicecfg " subjectAltName = DNS:$server, DNS:$thisfqdn"
	Add-Content $servicecfg "`n[ req_distinguished_name ]"
	Add-Content $servicecfg " countryName = $country"
	Add-Content $servicecfg " stateOrProvinceName = $state"
	Add-Content $servicecfg " localityName = $city"
	Add-Content $servicecfg " 0.organizationName = $org"
	Add-Content $servicecfg " organizationalUnitName = $service"
	Add-Content $servicecfg " commonName = $thisfqdn"

	&$openssl req -new -nodes -out $netappcsr -keyout $tempkey -config $servicecfg
	&$openssl rsa -in $tempkey -out $netappkey
	Remove-Item $tempkey
	certreq -submit -config "$rootCA\$rootCAName" -attrib "CertificateTemplate:WebServer" $netappcsr $netappcrt
	&$openssl pkcs12 -export -in $netappcrt -inkey $netappkey -certfile $root64 -name $keyalias -passout pass:netapp -out $netapppfx
	Get-Content $netappcrt > $chainpem; Get-Content $root64 >> $chainpem

###############################################################################
#
# NetApp Virtual Storage Console
#
###############################################################################

	Write-Host -Foreground "DarkBlue" -Background "White" "Updating NetApp Virtual Storage Console.."

	Stop-Service NVPF

	Write-Host -Foreground "DarkBlue" -Background "White" "Backing up current keystore.."
	$null = (New-Item -Type Directory $backupdir)
	Move-Item "$vscdir\etc\keystore.properties" $backupdir
	Move-Item "$vscdir\etc\nvpf.keystore" $backupdir

	Set-Content "$vscdir\etc\keystore.properties" "http.ssl.keystore.file=etc/nvpf.keystore"
	Add-Content "$vscdir\etc\keystore.properties" "http.ssl.keystore.password=netapp"
	Add-Content "$vscdir\etc\keystore.properties" "http.ssl.key.password=netapp"

	Write-Host -Foreground "DarkBlue" -Background "White" "Creating new NetApp Virtual Storage Console keystore.."
	$null = (&"$vscdir\jre\bin\keytool.exe" -v -importkeystore -srckeystore "$servicedir\netapp.pfx" -srcstoretype pkcs12 -srcstorepass netapp -srcalias "netapp" -destkeystore "$vscdir\etc\nvpf.keystore" -deststoretype JKS -deststorepass netapp -destkeypass netapp -destalias "netapp")
	$null = (&"$vscdir\jre\bin\keytool.exe" -alias "netapp" -noprompt -v -importcert -keystore "$vscdir\etc\nvpf.keystore" -deststoretype JKS -storepass netapp -file $netappcrt)

	Start-Service NVPF

Write-Host -Foreground "DarkBlue" -Background "White" "Done!"

Done!

I was surprised to find that that vSphere 4.1 and 5.0 are far more architecturally similar than 5.0 and 5.1. The 5.0 script required just one extra line of code to adjust for a different registry entry, then it worked very well on 4.1U3.

So without further ado, you can download ReplaceSSL-vSphere41-50.ps1, modify the variables as necessary and run it on each of your farm servers. This script requires you to modify just 10 variables as seen in the snippet below:

If you are interested in the approximate steps taken, you can browse the vSphere 5.1 SSL replacement post. Just be aware that the SSO section does not apply.

Below is a script I use in conjunction with my vSphere/PowerShell Replace SSL script.

This is the first time I've actually used PowerCLI so I'm unsure if this script follows Best Practices, but hey, it worked for me in my lab environment

"What it does.."

• Creates the certificate directory if it does not exist
• Logs into specified vSphere Server
• Automatically downloads Root64.cer from the CA's web service
• Downloads and extracts OpenSSL if the files do not exist in the specified path
• Generates all SSL certificates for each of the services on the server.

If $upsateesx is set to true..

• Downloads Putty SCP
• Checks to see if SSH is running on the esx host. If not, it temporarily enables it
• Prompts for and validates credentials
• Backs up all SSL Certs on the server
• Uploads the new certs
• Returns SSH to previous state

Once the new certs have been uploaded, you will have to restart the ESX host, or set it into maintenance mode and restart the Management services.

##############################################################################################
#
#   ESX Certificate Generation and Upload version 0.5
#   Tested on:  ESX 5.1 / vCenter 5.1U1 / PowerCLI 5.1 Release 2
#               ESX 4.1 / vCenter 4.1U3
#   No guarantees, warranties, etc.
#   Blog post: http://goo.gl/OdIlF
#
##############################################################################################

# vCenter Server FQDN
$vcserver = "vcenter41.base.local"

# It is recommended that you place the certs on a network location
$basedir = "\\fileserver\share\Certs"

# Enter your Windows Certificate Authority information
# below. Make sure your $rootCA responds to certutil and web requests.
$rootCA = "dc.base.local"
$rootCAName = "BASE-DC-CA"
$email = "vmware@base.local"
$org = "NetNerds"
$city = "Kaplan"
$state = "LA"
$country = "US"

# This can be WebServer or the VMware-SSL certificate
# template found here: http://goo.gl/m98FE
$certTemplate = "CertificateTemplate:WebServer"

# Enter the path of your openssl.exe (0.x and 1.x are supported).
# If you don't have OpenSSL already, the script will download it for you.
$openssldir = "C:\OpenSSL-Win32"
$openssl = $openssldir+"\bin\openssl.exe"

# Do you want the script to automatically backup the old ESX certs
# and upload the new certs to each esx host?
$updateesx = $true

##############################################################################################
#
#	You shouldn't need to change anything below.
#
##############################################################################################
if (!(Test-Path("$basedir"))) { $null = mkdir "$basedir" }

$backuptime = (get-date -uformat "%m%d%Y%H%M%S")
$esxhosts = @{}

Write-Host -Foreground "Black" -Background "White" "Logging into $vcserver."
if ($global:DefaultVIServers.Count -eq 0 -or ($global:DefaultVIServers).Name -ne $vcserver) {Connect-ViServer $vcserver}

Write-Host -Foreground "Black" -Background "White" "Getting list of esx servers."
$esxServers = (Get-VMHost).Name
foreach ($esxServer in $esxServers) {
	$esxdir = "$basedir\$esxServer-esx"
	$esxhosts.Add("$esxServer-esx", $esxServer)
}

Write-Host -Foreground "Black" -Background "White" "Downloading root CA Cert.."
$wc = New-Object System.Net.WebClient
$url = "http://$rootCA/certsrv/certnew.cer?ReqID=CACert&Renewal=0&Enc=b64"
$root64 = "$basedir\Root64.cer"
$wc.UseDefaultCredentials = $true
$wc.DownloadFile($url,$root64)

if (!(Test-Path($openssl))) {
	Write-Host -Foreground "Black" -Background "White" "Downloading OpenSSL.."
	$null = mkdir $openssldir
	$sslurl = "https://openssl-for-windows.googlecode.com/files/openssl-0.9.8k_WIN32.zip"
	$sslzip = "$env:temp\openssl.zip"
	$wc.DownloadFile($sslurl,$sslzip)
	$env:path = $env:path + ";$openssldir"

	Write-Host -Foreground "Black" -Background "White" "Extracting OpenSSL.."
	$shellApplication = new-object -com shell.application
	$zipPackage = $shellApplication.NameSpace($sslzip)
	$destinationFolder = $shellApplication.NameSpace($openssldir)
	$destinationFolder.CopyHere($zipPackage.Items())
	Remove-Item $sslzip
} 

$wc = New-Object System.Net.WebClient
if ($updateesx -eq $true) {
	Write-Host -Foreground "Black" -Background "White" "Downloading Putty SCP.."
	$scpurl = "http://tartarus.org/simon/20090227-kbdint-batch/x86/pscp.exe" # patched version for keyhost prompt issue
	$scp = "$env:temp\pscp.exe"
	$wc.DownloadFile($scpurl,$scp)
}

######################################################################
#
#	Generate Certs
#
######################################################################
Write-Host -Foreground "Black" -Background "White" "Generating service certs.."
foreach ($esxhost in $esxhosts.GetEnumerator()) {
	$service = $esxhost.Name
	$esxserverfqdn = $esxhost.Value
	$esxserver = $esxserverfqdn.Substring(0,$esxserverfqdn.IndexOf("."))

	$servicedir = "$basedir\$service"
	$servicecfg = "$servicedir\$service.cfg"
	$tempkey = "$servicedir\temp.key"
	$ruikey = "$servicedir\rui.key"
	$ruicsr = "$servicedir\rui.csr"
	$ruicrt = "$servicedir\rui.crt"
	$ruipfx = "$servicedir\rui.pfx"
	$chainpem = "$servicedir\chain.pem"
	$backupdir = "$servicedir\backup-$backuptime"
	$keyalias = "rui" 

	if (Test-Path($servicedir)) { $null = Remove-Item "$servicedir\*.*" } else {$null = mkdir $servicedir }

	Set-Content $servicecfg "[ req ]"
	Add-Content $servicecfg " default_md = sha512"
	Add-Content $servicecfg " default_bits = 2048"
	Add-Content $servicecfg " default_keyfile = rui.key"
	Add-Content $servicecfg " distinguished_name = req_distinguished_name"
	Add-Content $servicecfg " encrypt_key = no"
	Add-Content $servicecfg " prompt = no"
	Add-Content $servicecfg " string_mask = nombstr"
	Add-Content $servicecfg " req_extensions = v3_req"
	Add-Content $servicecfg "`n[ v3_req ]"
	Add-Content $servicecfg " basicConstraints = CA:FALSE"
	Add-Content $servicecfg " keyUsage = digitalSignature, keyEncipherment, dataEncipherment"
	Add-Content $servicecfg " extendedKeyUsage = serverAuth"
	Add-Content $servicecfg " subjectAltName = DNS:$esxserver, DNS:$esxserverfqdn"
	Add-Content $servicecfg "`n[ req_distinguished_name ]"
	Add-Content $servicecfg " countryName = $country"
	Add-Content $servicecfg " stateOrProvinceName = $state"
	Add-Content $servicecfg " localityName = $city"
	Add-Content $servicecfg " 0.organizationName = $org"
	Add-Content $servicecfg " organizationalUnitName = $service"
	Add-Content $servicecfg " commonName = $esxserverfqdn"

	&$openssl req -new -nodes -out $ruicsr -keyout $tempkey -config $servicecfg
	&$openssl rsa -in $tempkey -out $ruikey
	Remove-Item $tempkey
	certreq -submit -config "$rootCA\$rootCAName" -attrib $certTemplate $ruicsr $ruicrt
	&$openssl pkcs12 -export -in $ruicrt -inkey $ruikey -certfile $root64 -name $keyalias -passout pass:testpassword -out $ruipfx
	Get-Content $ruicrt > $chainpem; Get-Content $root64 >> $chainpem

	### Start ESX cert upload if updateesx is true and certificate generation is successful
	if ($updateesx -eq $true -and (Test-Path($ruikey)) -and (Test-Path($ruicrt))) {
		$disablessh = $null; $failedauth = 0
		$sshservice = (Get-VMHostService -VMHost $esxserverfqdn -Server $vcserver | Where { $_.Key -eq "TSM-SSH"})

		if ($sshservice.Running -eq $false) {
			Write-Host -Foreground "Black" -Background "White" "Temporarily enabling SSH on $esxserverfqdn" ; $disablessh = $true
			$null = Start-VMHostService -HostService $sshservice -Confirm:$false
		}

		Write-Host -Foreground "Black" -Background "White" "Validating authentication."
		Write-Host -Foreground "Black" -Background "White" "You can ignore any SSH keyhost prompts you may see.."
		do {
				$msg = "Enter the username and password for $esxserverfqdn";
				$creds = $Host.UI.PromptForCredential($caption,$msg,"root",$domain)
				$esxusername = $creds.username;	$esxpassword = $creds.GetNetworkCredential().password
				$esxsslpath = "$esxusername@$esxserverfqdn"+":/etc/vmware/ssl/"
				$authenticated = $null
				$checkauth = (Echo "Y" | &($scp) -scp -pw $esxpassword -ls $esxsslpath)

				if ($checkauth -eq $null) {
					$authenticated = $false
					$failedauth++
				}
			}	until ($authenticated -ne $false -or $failedauth -gt 4)	

		if ($failedauth -gt 4) { Write-Host -Foreground "Black" -Background "White" "Sorry, too many failed logins."; Break }
		Write-Host -Foreground "Black" -Background "White" "`rAuthentication accepted!"

		Write-Host -Foreground "Black" -Background "White" "Backing up current certs.."
		$null = (New-Item -Type Directory $backupdir)
		echo "Y" | &($scp) -scp -batch -pw $esxpassword "$esxsslpath/rui.key" $backupdir
		echo "Y" | &($scp) -scp -batch -pw $esxpassword "$esxsslpath/rui.crt" $backupdir

		Write-Host -Foreground "Black" -Background "White" "Uploading new certs.."
		echo "Y" | &($scp) -scp -batch -pw $esxpassword "$ruikey" $esxsslpath
		echo "Y" | &($scp) -scp -batch -pw $esxpassword "$ruicrt" $esxsslpath

		if ($disablessh) {
			Write-Host -Foreground "Black" -Background "White" "Returning SSH to disabled state on $esxserverfqdn"
			$null = Stop-VMHostService -HostService $sshservice -Confirm:$false
		}

		Write-Host -Foreground "Black" -Background "White" "Finished uploading files on $esxserverfqdn. Reboot the ESX host to activate new certificates."
	}

}

if ($updateesx -eq $true) { $null = Remove-Item $scp }

Alternatively, you can download the .ps1 file from here.

Note that you will have to re-add ESX to vCenter because the host's SSL thumbprint has changed. Regarding updating ESX's SSL, Derek Seaman suggests:

If your ESXi host is already managed by vCenter, the HA agent can get very confused by the new SSL certificate thumbprint. I would strongly suggest you first put your host in maintenance mode, remove it from the vCenter inventory, update the SSL certificate, reboot the ESXi host, then re-add it to the vCenter inventory.

At first, I attempted to use strictly Windows commands (certutil, certreq, etc) and PowerShell, but eventually gave in and incorporated OpenSSL into my script. Generating the certs were just the beginning, though.

Replacing the certificates in an automated fashion and getting each service to behave after the change was an extremely time-consuming task. VMware's documentation and KB articles leave a lot of room for improvement, but fortunately, David Seaman's blog was able to provide a lot of information that was either easy to miss, or missing entirely.

Numerous articles suggested using VMware's Certificate Automation Tool but the tool wasn't automated enough for my liking, even with supplemental scripts provided by other bloggers. I looked into guts of the Certificate Automation tool and, after a good bit of trial and error, replicated many of its techniques using PowerShell. Using these techniques, and following the suggestions found on forums and blogs, I was able to create a script that can replace the SSL certs of all of my vSphere lab servers in under 20 minutes, a majority of which is spent watching PowerShell stop and start services.

This script requires you to enter less than 15 variables as seen in the snippet below:

You can see that the SSO admin username and password are in plain text. Unlike vCenter credentials, there was no easy way to validate the SSO username/password and the pros of placing the username and password there in plain text outweighed the cons. vCenter credentials were easier to validate and more of a priority for me to protect since they're usually Windows credentials.

Also, note that the default "Web Server" SSL certificate template is no longer adequate. Please visit Derek Seaman's blog for instructions on how to create a certificate template which will work for all of the vSphere services.

"What it does.."

Start up

• If the server running the script is not the SSO server, it ensures the remote SSO Server's SSL certs have been updated first
• Checks the registry to see which vSphere services exist on the server running the script and sets service variables
• Creates the certificate directory if it does not exist
• Backs up all SSL Certs on the server
• Validates vCenter authentication if vCenter or VUM exist on the server
• Automatically downloads Root64.cer from the CA's web service
• Downloads and extracts OpenSSL if the files do not exist in the specified path
• Generates all SSL certificates for each of the services on the server. Uses server name + service name as the OU so that each cert can be distinguished.

If SSO service exists

• Stops SSO Service
• Generates new SSO keystore using the newly created SSO SSL certificate
• Copies Root64.cer to %programdata%\VMware\SSL\ca_certificates.cer
• Creates new hash file in %programdata%\VMware\SSL
• Updates SSO using rsautil.cmd
• Starts SSO Service
• Automatically builds service.properties and service_id files and stores them in %programdata%\VMware\ServiceIDs
• Reregisters all services using new root certificate
• Restarts SSO, and if they exist Log Browser, Web Client and Inventory services.

If Inventory service exists

• Unregisters Inventory service with SSO
• Stops Inventory service
• Copies new certs to the Inventory service SSL directory
• Starts Inventory Service
• Registers Inventory service with SSO

If vCenter service exists

• Copies new certs to the vCenter service SSL directory
• Using credentials previously entered, logs into vCenter service's mob website to automatically invoke reloadSslCertificate
• Restarts all vCenter related services
• Reregisters vCenter with Inventory Service

If WebClient services exists

• Stops WebClient and LogBrowser services
• Removes all files from SerenityDB directory
• Copies new certs to the Web Client and Log Browser service SSL directories
• Stops vCenter and Inventory Services if they exist on the local server
• Restarts SSO service on local or remote server
• Starts vCenter and Inventory Services if they exist on the local server
• Starts WebClient and LogBrowser services

If Update Manager exists

• Stops Update Manager services
• Copies new certs to the Update Manager service SSL directory
• Generates new Update Manager keystore using the newly minted Update Manager Certificates
• Updates registry entry with keystore password (testpassword)
• Runs vciInstallUtils to update VUM using credentials previously entered
• Starts Update Manager services

If Orchestrator exists

• Copies new certs to the Orchestrator service SSL directory
• Stops services if necessary
• Generates new Orchestrator keystore using newly created Orchestrator certificates
• Adds SSO Certificate to keystore
• Restarts Orchestrator services then returns them to their previous state of Running or Stopped

If you have vCenter servers in linked mode and are running the Web Client, you may run into the error message "Cannot connect to Inventory Service on [server]" when logging into the Web Client. I have not found a predictable way to fix this. Usually, it can be solved by first restarting the server running Web Client, then restarting the vCenter/Inventory server.

This script also has other limitations, many of them similar to the Certificate Automation Tool.

Limitations

Limitations specific to this script

• Only uses Windows Domain Certificate Authorities
• Does not account for intermediary CAs
• Has not been tested in large environments with HA and DRS
• Has not been tested with environments running: VMware Site Recovery Manager, vSphere Data Recovery, vCloud Director, or third-party solutions
• Does not have a rollback feature, yet. For rollback, I relied on Snapshots and database backups.
• Does not have advanced logging
• I don't recommend running this in a production environment until it's been vetted by far more people

Limitations that exist in VMware's tool that likely exist within this script

• vCenter Single Sign-On Password cannot contain spaces
• vCenter Orchestrator may fail to connect when using multiple vCenter Servers.
  • You can update add additional vCenter Server SSL certificates using the VMO Configuration Webpage (https://vmoserver:8283/ default login: vmware/vmware -> vCenter Server -> SSL Certificates.
  • Add all vCenter Certificates found in your Certs directory.
  • Note that if vCenter and VMO are running on the same server, the vCenter cert will be automatically added.
• Client Not authenticated error when connecting to VMware Inventory service in Linked Mode Configurations. Wait 10 minutes and this should resolve itself.

Why I prefer using this script over VMware's

• Requires minimal information and interaction
  • Automatically downloads OpenSSL if neccessary
  • Automatically generates the certificates based off of a few variables
  • Automatically detects services and runs the SSL updates in the necessary order without user intervention
  • If vCenter or VUM exists on the server, you wil be prompted for your vCenter credentials. This is the extent of interaction that the script will require:

  

• Replaces all of the same certificates: SSO, Inventory Service, vCenter, Update Manager, Web Client/Log Browser, and Orchestrator
• Also, works on vSphere farms with multiple servers (you must update the SSO server first)
• It's all contained in just one (nearly 600 line) script
• Works on 5.1 and 5.1U1

In the end, your Cert collection will look something like this:

* Note that the esx certificate output was created using this script.

And each of your services will be encrypted with trusted certificates:

     


     

And, of course, Web Client, after a couple reboots.

Getting started

1. Ensure your Windows Domain CA certificate is trusted by members of your domain
2. Take a snapshot of each vSphere server on which you will run this script
3. Backup each of your databases
4. Find a secure location on the network to store your certs (ie. \\fileserver\share\Certs)
5. Visit Derek Seaman's blog and create a new certificate template
6. Shut down the following services if they exist: VMware Site Recovery Manager, vSphere Data Recovery, vCloud Director, third-party solutions that connect to vCenter.
7. Download the ReplaceSSL-vSphere51.ps1 script
8. Change the variables
9. Run the script first on the server running the SSO
10. Run the script on all other servers
11. If the Web Client connects to multiple vCenter servers, reboot the server running Web Client, as well as the server(s) running Inventory Service and vCenter.
12. Consider running the complimentary ESX Script

Once the scripts are complete, you can visit each of your sites to confirm the SSL Certificates have been replaced. Please note that Log Browser and the Web Client take up to 5 minutes to fully restart.

Finally, bask in the glory of your trusted SSL certificates:

This code imports events from the last 65 minutes. For the initial import, set $xml to wevtutil.exe qe ForwardedEvents /e:Events. As an aside, I was surprised to see that wevtutil is FAR faster than PowerShell's Get-WinEvent, especially during the initial import of a large logs.

PS C:\Scripts> Measure-Command {c:\scripts\final-getwinevent.ps1}
Days              : 0
Hours             : 0
Minutes           : 1
Seconds           : 19
Milliseconds      : 293
Ticks             : 792930218
TotalDays         : 0.00091774330787037
TotalHours        : 0.0220258393888889
TotalMinutes      : 1.32155036333333
TotalSeconds      : 79.2930218
TotalMilliseconds : 79293.0218

PS C:\Scripts> Measure-Command {c:\scripts\final-wevtutil.ps1}
Days              : 0
Hours             : 0
Minutes           : 0
Seconds           : 4
Milliseconds      : 957
Ticks             : 49571333
TotalDays         : 5.73742280092593E-05
TotalHours        : 0.00137698147222222
TotalMinutes      : 0.0826188883333333
TotalSeconds      : 4.9571333
TotalMilliseconds : 4957.1333

From 79 seconds to 5 for 5500 records! Looks like having to rewrite this was a good thing, after all.

There are a lot of blog posts out there that use PowerShell or some other scheduled script to perform the backups, but it occurred to me the other day that I can totally run all of my backup and maintenance scripts from another SQL Server's SQL Agent. Duh! The key is to use the Connections tab to connect to other SQL instances.

I'll be honest, because these servers may come and go and their databases are unpredictable, I don't want to install anything local like Ola Hallengren's Backup and Maintenance Scripts, so I setup simple Maintenance Plans with the following parameters:

• I made one plan with multiple subplans.
• Weekly: Full backup, integrity checks, index rebuild, Maintenance cleanup task (4 weeks), History cleanup task (4 weeks)
• Daily: Differential, Maintenance cleanup task (4 weeks)
• Staggered schedules between server backups -- I don't expect SQL Express databases to take any longer than 30 minutes, so every instance backup is scheduled 30 minutes apart.
• The backups go to the same network share as the rest of the SQL instances which kinda look like this: \\sqlbackupserver\backups\HOSTNAME\full, \\sqlbackupserver\backups\HOSTNAME\diff.
• No transaction log backups are scheduled -- so far, I haven't found a SQL Express instance with databases that are set to anything but the SIMPLE recovery model.

Make sure the SQL Server Agent account has adequate access to the SQL Server Express, and that you don't check "Compress backups." Otherwise, your backups will fail with a Backup Error: 3041, Severity: 16, State: 1 error code as SQL Express doesn't support compressed backups.

I always forget about Multiserver Administration. I've actually never worked in an environment that uses it, even though it seems to have a lot of potential. I think one of the biggest reasons is that most organizations do not use SSL encryption between SQL Servers, yet out of the box, Multiserver Administration requires SSL encryption for communication between the master and the targets.

Want to change this option? You'll have to modify the registry. Come on, Microsoft: nobody wants to touch a production SQL Server's registry and I think this is the biggest roadblock to mass adoption of Multiserver Administration.

The registry subkey that needs to be changed is MsxEncryptChannelOptions. There isn't a whole lot of information about this subkey (such as what other services it impacts) but I'm hoping that since its prefixed with "Msx" that and sits in the SQLAgent key, the change will be isolated to Multiserver Administration. So here are the 3 options:

Like the table says, 2 (Encryption+SSL required) is the default. Most blogs I've seen change their option to 0 (No encryption), but I tested it with 1 (Encryption enabled+ SSL not required) and default out of the box SQL encryption settings and it worked. Microsoft says this about the default encryption:

Credentials (in the login packet) that are transmitted when a client application connects to SQL Server are always encrypted. SQL Server will use a certificate from a trusted certification authority if available. If a trusted certificate is not installed, SQL Server will generate a self-signed certificate when the instance is started, and use the self-signed certificate to encrypt the credentials.

I always prefer encryption if it's not disruptive, so this is the setting I will recommend, and the setting that is default in the script below. This script asks for the SQL Server version (SQL2k5 is not supported because I no longer use it and instance paths are more challenging.)

(Thanks to quickclix for the easy PowerShell menu code.)

Once you've run this script and modified the settings on your Target servers, you can easily setup Multiserver Administration. Note that the default setting may create a SQL Server login for the target server automatically. I uncheck that option because I'm trying to get away from local SQL Server logins and all of my SQL Agents run under the same domain account anyway.

Ultimately, the only thing that worked was PowerShell's Get-WinEvent cmdlet. And then, it only worked in one specific case for me -- if the events are collected and parsed on a Windows 2012 server. As of today, there's an unresolved bug in Get-WinEvent that often results in NULL LevelDisplayName, Message, and TaskDisplayName columns. I copied the exact code below on a Win2k8 R2 server and a Win 8 workstation and ran into the NULLs issue repeatedly. Your results may vary, however, as some users have reported success by tweaking a few things in Win2k8 R2 Server.

So, fire up a Windows 2012 box, setup your SQL Server and let's get started:

The SQL Part

After looking at the data returned by Get-WinEvent, I found the following columns to be the most useful: ID, LevelDisplayName, LogName, MachineName, Message, ProviderName, RecordID, TaskDisplayName, TimeCreated. Then I created a table using those columns:

In order to avoid duplicates during the hourly imports, I created the table using a unique index with IGNORE_DUP_KEY = ON on 3 columns: RecordID, MachineName and LogName.

Next I had to decide how I'd get the data from PowerShell into SQL Server. After reading up on sqlservercentral.com and technet, I decided on hourly imports using sqlbulkcopy.

The PowerShell Part

Forwarded Events are a tricky thing. For some reason, the way that one would usually filter Get-WinEvent results using FilterHasTable kept returning the result Get-WinEvent : No events were found that match the specified selection criteria. I found a number of others who ran into this issue, too and similar errors occurred when people attempted to use LogParser. After all that, I didn't have much hope in FilterXML working, but it actually did! So we're going to use that after we perform our initial import.

Here's the code for the initial import which gathers ALL events in Forwarded Events.

You may noticed that I manually built a datatable instead of using Out-DataTable.ps1, which appears to be a fan favorite. I felt the code above kept things a little more tidy and the performance is still quite good.

Since Event Collection is an on-going thing, you'll likely want to import them on a regular basis. I built the necessary XML query by right clicking on Forwarded Events in Event Viewer -> Filter Current Log... -> Logged: (Change to one hour) -> Click XML tab at top -> Copy/Paste -> Voila.

Actually, using the syntax of this query, I figured out the syntax for FilterHashTable but having the GUI build my query makes it easy, so I stuck with that. Here is the code for the hourly import that you can setup in Task Scheduler.

With any luck, your SQL output should look something like this:

Woo.

EDIT: If you care about speed, check out this post where I write about using wevtutil instead of Get-WinEvent.

Things you'll need: Windows Server 2012 or 2008 R2, the appropriate ports open on your firewall if you're running one, access to a Domain Admin service account or the ability to add a Machine account to a local group on each of the servers you're collecting from.

So here's how to setup a Windows 2012 or 2008 R2 Server as the Event Collection Server: first, open up Event Viewer, right click on Forwarded Events and click Properties.

Note that Application, Security and System look a bit different than the others. I believe that's because they are considered Classic Windows Events. Classic Windows Events can be easily parsed in PowerShell using Get-EventLog and WMI's Win32_NTEventLogFile but the newer Event Types are a bit trickier.

The first thing you see in the Forwarded Events property tab is this:

Note the location of the ForwardedEvents.evtx file as it may come in handy for future troubleshooting or archiving. Next, click on the "Subscriptions" tab, then Create.

Now here, I will create just one generic Subscription to capture all Critical and error events. You can create multiple subscriptions for differing criteria and just send all of them to the Forwarded Events log. So select "Collector Initiated" and "Select Computers." Unfortunately, you have to add each server one by one when using the GUI. If you want to add in bulk, you'll likely have to do it using Group Policy and Source Initiation. I haven't found a way to do it in PowerShell yet.

Click Select Events and select what you'd like. My servers are super verbose, and I really only care about Criticals and Errors so I select those two Event Levels, and then By log: Application and System.

Once you're done with your Event selections, click on Advanced to modify the account you'd like to use as the collector (in my lab, I used a Domain Admin account).

If you use a Machine account or a non-admin, you'll have to go to each of the forwarding servers and add the Computer or User AD account to Event Log Readers.

Here, you can also configure the Event Delivery Optimization. I left it default for now, though later I do plan to explore delivering Events over HTTPS. Click OK until you're back at Event Viewer. Wait a few minutes and the Events should start coming in. In my experience, for each new computer I setup, I'll get one Information event from the Microsoft-Windows-EventForwarder provider. The event itself is kind of useless, saying "the description for Event 111 cannot be found." but I've learned to accept it's the Event Forwarding confirming there's a new computer in the subscription.

When things get going, your Windows Forwarding collection should look something like this:

In this screenshot, I've highlighted Attach Task to Event which allows you to run a program or send an email when the event occurs. Pretty cool.

If you need additional help, Microsoft has a Quick and Dirty Large Scale Eventing for Windows guide that is good for troubleshooting.

Initially, I set up my RD Gateway using too many Remote Desktop Services: Remote Desktop Connection Broker, Remote Desktop Gateway & Remote Desktop Web Access, but that was because was lead astray by Windows 2012's new GUI. Now, I've narrowed it down only to RD Gateway and I'm even fond of Metro (:O)

So to get this going, all you have to do is install and configure the Remote Desktop Gateway Services (RD Gateway) Role. That seems obvious, but Server Manager's interface which prominently displays an unconfigured "Remote Desktop Services" tab made me think I was missing something.

During the Role installation do: Role-based or feature-based installation -> Remote Desktop Services -> Remote Desktop Gateway

Then click Next a bunch of times. Something odd, when it asks you "Do you need an alternate source path?", even if you have the Windows Server 2012 ISO attached, you'll still need to click "Specify an alternate source path" and enter D:\sources\sxs (assuming your ISO is attached to D:)

Click Install and wait for the installation to complete. Now it's time to configure RD Gateway.

OPTIONAL: If you're on a domain with a Certificate Authority, you'll want to configure IIS to use a Domain Certificate. Open IIS Manager -> Select your server -> Server Certificates -> Create Domain Certificate. For "Common Name" make sure you enter your external FQDN. Note: I chose to go with dyndns.org since I have a dynamic IP. It's required that you use an externally resolvable hostname, otherwise Remote Desktop will fail if you try to use an IP or mismatched hosts.

Now, you'll need to configure RD Gateway. Go to Server Manager -> Tasks -> RD Gateway Manager.

Click View or modify certificate properties. If you don't have a Domain Certificate, just click Create and import certificate and ensure you use your external FQDN for the certificate name. Otherwise, choose Select an existing certificate.... Choose your certificate

Click Import -> Apply. Now that you're back at the RD Gateway Manager, expand the tree under your server name. Click Policies then on the right, click Create Authorization Polices for RD Gateway. Create an RD CAP and RD RAP (Recommended). In the name field, you can enter whatever you'd like. I chose "Default" -> Next -> Add Group -> Domain Admins -> (leave Client Computer blank)

Next, you'll be given the option to Enable or Disable Device Redirection. I just choose the default (all clients) and click Next -> Next -> Next - Default -> Next -> Allow users to connect to any network resource -> Next -> Allow Connections only to port 3389 -> Next -> Finish

Finally, open up Services and Start Remote Desktop Gateway

Voila! Now you can go modify your router rules to connect port 443 to your RD Gateway Server and/or read the important notes below.

A few important things to note
As an added security pre-caution, I went into IIS and disabled Anonymous access to my root IIS folder and ensured Windows Authentication was still enabled for the RPC folders.

Configuring the Remote Desktop Client is easy. Open up your Remote Desktop Client -> Advanced -> (Connect from Anywhere) Settings.

Enter the external hostname that you entered earlier during the configuration of RD Gateway. Go back to the general tab, and enter the FQDN of the domain server you wish to connect to. Don't worry about resolving the hostname if you're using an external DNS server -- DNS is resolved at the RD Gateway so if the RD Gateway can resolve the hostname, you're set.

If you choose to use a self-signed cert or you are attempting to connect from a computer that's not on the domain, you'll have to import the SSL cert to your Trusted Root Certification Authority. Otherwise, you'll receive the error "This computer can't verify the identity of the RD Gateway 'sample.server.com'. It's not safe to connect to servers that are not identified. Contact your network administrator for assistance."

There are a few ways to do this, but here's how I do it. I use Chrome to hit my server (ex. https://myserver.dyndns.org)

Click Certificate Information -> Details -> Copy to File. Save the cert, then find it using Windows Explorer. Right-click on the cert -> Install Certificate -> Place all Certificate in the Following Store -> Trusted Root Certification Authority -> Next -> Finish -> Yes.

You should now be able to connect and securely manage your network, all over SSL