Use nats-server blob instead of downloading gnatsd

The nats-server binary replaced the legacy forked gnatsd binary in 2022:
#2391

However, the integration specs were still using an old version of gnatsd
from an S3 bucket.

Now, the integration specs use the current nats-server blob directly.
This required that the nats-sync process also be launched during
integration tests in order to allow agents to connect. Additionally,
there was a race condition between an agent being launched by the Dummy
CPI and the nats-sync process adding the agent to the nats-server
config.

src/bosh-director/lib/cloud/dummy.rb

@@ -123,6 +123,14 @@ def create_vm(agent_id, stemcell_id, cloud_properties, networks, disk_cids, env)
           agent_process_agent_id = 'unresponsive-agent-fake-id-' + SecureRandom.uuid
         end
 
+
+        # Sleep long enough for the NATS sync process to add the agent ID to the NATS server config.
+        # If the agent attempts to connect before this, then the NATS server will forceably close
+        # the connection, the agent will not attempt to reconnect, and will eventually shut down.
+        # On a real VM, the sv monitoring service would restart the agent, but there's no such
+        # service here.
+        sleep 3
+
         agent_pid = spawn_agent_process(agent_process_agent_id, cloud_properties['legacy_agent_path'])
         vm = VM.new(agent_pid.to_s, agent_id, cloud_properties, ips)
 

src/spec/assets/sandbox/bosh_nats_sync_config.yml.erb

@@ -0,0 +1,44 @@
+<%=
+
+authentication = if user_authentication == 'uaa'
+  # Pulled from src/spec/assets/uaa_config/asymmetric/uaa.yml
+  {
+    'user' => '',
+    'password' => '',
+    'client_id' => 'nats-sync',
+    'client_secret' => 'secret'
+  }
+else
+  # Pulled from src/spec/assets/sandbox/director_test.yml.erb
+  {
+    'user' => 'test',
+    'password' => 'test',
+    'client_id' => '',
+    'client_secret' => ''
+  }
+end
+
+director = authentication.merge({
+  'url' => director_url,
+  'ca_cert' => uaa_ca_cert_path,
+  'director_subject_file' => nats_sync_director_subject_file_path,
+  'hm_subject_file' => nats_sync_hm_subject_file_path,
+})
+
+params = {
+  'director' => director,
+  'intervals' => {
+    # Set to a low value to ensure VMs are authorized before they connect.
+    'poll_user_sync' => 1,
+  },
+  'nats' => {
+    'config_file_path' => nats_sync_auth_json_path,
+    'nats_server_executable' => nats_server_executable_path,
+    'nats_server_pid_file' => nats_server_pid_path,
+  },
+  'logfile' => nats_sync_log_path,
+}
+
+YAML.dump(params)
+
+%>

src/spec/assets/sandbox/nats.conf.erb

@@ -1,49 +1,38 @@
-# NATS Configuration file
-
-listen: localhost:<%= nats_port %> # host/port to listen for client connections
+host: localhost
+port: <%= nats_port %>
+logtime: true
 
 log_file: "<%= nats_log_path %>"
 
 authorization {
-  DIRECTOR_PERMISSIONS: {
-    publish: [
-      "agent.*",
-      "hm.director.alert"
-    ]
-    subscribe: ["director.>"]
-  }
-
-  AGENT_PERMISSIONS: {
-    publish: [
-      "hm.agent.heartbeat._CLIENT_ID",
-      "hm.agent.alert._CLIENT_ID",
-      "hm.agent.shutdown._CLIENT_ID",
-      "director.*._CLIENT_ID.*"
-    ]
-    subscribe: ["agent._CLIENT_ID"]
-  }
-
-  HM_PERMISSIONS: {
-    publish: []
-    subscribe: [
-      "hm.agent.heartbeat.*",
-      "hm.agent.alert.*",
-      "hm.agent.shutdown.*",
-      "hm.director.alert"
-    ]
-  }
 
-  FULL_PERMISSIONS: {
-    publish: [">"]
-    subscribe: [">"]
-  }
-
-  certificate_clients: [
-    {client_name: director.bosh-internal, permissions: $DIRECTOR_PERMISSIONS},
-    {client_name: agent.bosh-internal, permissions: $AGENT_PERMISSIONS},
-    {client_name: bootstrap.agent.bosh-internal, permissions: $AGENT_PERMISSIONS},
-    {client_name: hm.bosh-internal, permissions: $HM_PERMISSIONS},
-    {client_name: integration.test.bosh-internal, permissions: $FULL_PERMISSIONS},
+  users = [
+    {
+      user: "C=USA, O=Cloud Foundry, CN=default.director.bosh-internal"
+      permissions: {
+        publish: [ "agent.*", "hm.director.alert" ]
+        subscribe: [ "director.>" ]
+      }
+    },
+    {
+      user: "C=USA, O=Cloud Foundry, CN=default.hm.bosh-internal"
+      permissions: {
+        publish: []
+        subscribe: [
+          "hm.agent.heartbeat.*",
+          "hm.agent.alert.*",
+          "hm.agent.shutdown.*",
+          "hm.director.alert"
+       ]
+      }
+    },
+    {
+      user: "C=USA, O=Cloud Foundry, CN=default.integration.test.bosh-internal"
+      permissions: {
+        publish: [">"]
+        subscribe: [">"]
+      }
+    }
   ]
 
   timeout: 5
@@ -53,7 +42,12 @@ tls {
   cert_file: "<%= nats_certificate_paths['server']['certificate_path'] %>"
   key_file:  "<%= nats_certificate_paths['server']['private_key_path'] %>"
   ca_file:   "<%= nats_certificate_paths['ca_path'] %>"
-  verify:    true
+  verify_and_map: true
   timeout: 5
-  enable_cert_authorization: true
 }
+
+ping_interval: 5s
+ping_max: 2
+max_payload: 1048576
+
+include ./auth.json

src/spec/assets/uaa_config/asymmetric/uaa.yml

@@ -106,6 +106,14 @@ uaa:
       authorities: uaa.none,bosh.admin
       secret: "secret"
       redirect-uri: http://127.0.0.1
+    nats-sync:
+      id: nats-sync
+      override: true
+      authorized-grant-types: client_credentials
+      scope: ''
+      authorities: bosh.admin
+      secret: "secret"
+      redirect-uri: http://127.0.0.1
     production_team:
       id: production_team
       override: true

src/spec/integration_support/gnatsd_manager.rb

@@ -1,5 +1,3 @@
-require 'integration_support/artifact_installer'
-
 module IntegrationSupport
   class GnatsdManager
     def self.install
@@ -11,19 +9,43 @@ def self.executable_path
     end
 
     def self.installer
-      @installer ||=
-        ArtifactInstaller.new(
-          File.join('tmp', 'gnatsd'),
-          'gnatsd',
-          {
-            version: '1.3.0-bosh.10',
-            darwin_sha256: 'fac87b6b9b46830551f32f22930a61e2162edf025304f0f2ce7282b4350003f7',
-            linux_sha256: 'e5362a7c88ed92d4f4263b1b725e901fe29da220c3548e37570793776b5f6d51',
-            bucket_name: 'bosh-gnatsd',
-          }
-        )
+      @installer ||= NatsServerBlobInstaller.new
     end
 
     private_class_method :installer
   end
+
+  class NatsServerBlobInstaller
+    INSTALL_DIR = File.join(IntegrationSupport::Constants::BOSH_REPO_SRC_DIR, 'tmp', 'integration-nats')
+
+    def install
+      Dir.chdir(IntegrationSupport::Constants::BOSH_REPO_ROOT) do
+        run_command("mkdir -p #{INSTALL_DIR}")
+        run_command('bosh sync-blobs')
+        run_command('tar -zxvf blobs/nats/nats-server-*.tar.gz -C /tmp')
+        run_command("cp /tmp/nats-server-*/nats-server #{executable_path}")
+        run_command("chmod +x #{executable_path}")
+      end
+    end
+
+    def executable_path
+      File.join(INSTALL_DIR, 'nats-server')
+    end
+
+    private
+
+    def run_command(command, environment = {})
+      io = IO.popen([environment, 'bash', '-c', command])
+
+      lines =
+        io.each_with_object("") do |line, collect|
+          collect << line
+          puts line.chomp
+        end
+
+      io.close
+
+      lines
+    end
+  end
 end