diff --git a/src/bosh-director/lib/cloud/dummy.rb b/src/bosh-director/lib/cloud/dummy.rb index 3b67845c159..34bd706b176 100644 --- a/src/bosh-director/lib/cloud/dummy.rb +++ b/src/bosh-director/lib/cloud/dummy.rb @@ -123,6 +123,14 @@ def create_vm(agent_id, stemcell_id, cloud_properties, networks, disk_cids, env) agent_process_agent_id = 'unresponsive-agent-fake-id-' + SecureRandom.uuid end + + # Sleep long enough for the NATS sync process to add the agent ID to the NATS server config. + # If the agent attempts to connect before this, then the NATS server will forceably close + # the connection, the agent will not attempt to reconnect, and will eventually shut down. + # On a real VM, the sv monitoring service would restart the agent, but there's no such + # service here. + sleep 3 + agent_pid = spawn_agent_process(agent_process_agent_id, cloud_properties['legacy_agent_path']) vm = VM.new(agent_pid.to_s, agent_id, cloud_properties, ips) diff --git a/src/spec/assets/sandbox/bosh_nats_sync_config.yml.erb b/src/spec/assets/sandbox/bosh_nats_sync_config.yml.erb new file mode 100644 index 00000000000..278198ac14d --- /dev/null +++ b/src/spec/assets/sandbox/bosh_nats_sync_config.yml.erb @@ -0,0 +1,44 @@ +<%= + +authentication = if user_authentication == 'uaa' + # Pulled from src/spec/assets/uaa_config/asymmetric/uaa.yml + { + 'user' => '', + 'password' => '', + 'client_id' => 'nats-sync', + 'client_secret' => 'secret' + } +else + # Pulled from src/spec/assets/sandbox/director_test.yml.erb + { + 'user' => 'test', + 'password' => 'test', + 'client_id' => '', + 'client_secret' => '' + } +end + +director = authentication.merge({ + 'url' => director_url, + 'ca_cert' => uaa_ca_cert_path, + 'director_subject_file' => nats_sync_director_subject_file_path, + 'hm_subject_file' => nats_sync_hm_subject_file_path, +}) + +params = { + 'director' => director, + 'intervals' => { + # Set to a low value to ensure VMs are authorized before they connect. + 'poll_user_sync' => 1, + }, + 'nats' => { + 'config_file_path' => nats_sync_auth_json_path, + 'nats_server_executable' => nats_server_executable_path, + 'nats_server_pid_file' => nats_server_pid_path, + }, + 'logfile' => nats_sync_log_path, +} + +YAML.dump(params) + +%> \ No newline at end of file diff --git a/src/spec/assets/sandbox/nats.conf.erb b/src/spec/assets/sandbox/nats.conf.erb index 719eec9dd38..af730cc5c92 100644 --- a/src/spec/assets/sandbox/nats.conf.erb +++ b/src/spec/assets/sandbox/nats.conf.erb @@ -1,49 +1,38 @@ -# NATS Configuration file - -listen: localhost:<%= nats_port %> # host/port to listen for client connections +host: localhost +port: <%= nats_port %> +logtime: true log_file: "<%= nats_log_path %>" authorization { - DIRECTOR_PERMISSIONS: { - publish: [ - "agent.*", - "hm.director.alert" - ] - subscribe: ["director.>"] - } - - AGENT_PERMISSIONS: { - publish: [ - "hm.agent.heartbeat._CLIENT_ID", - "hm.agent.alert._CLIENT_ID", - "hm.agent.shutdown._CLIENT_ID", - "director.*._CLIENT_ID.*" - ] - subscribe: ["agent._CLIENT_ID"] - } - - HM_PERMISSIONS: { - publish: [] - subscribe: [ - "hm.agent.heartbeat.*", - "hm.agent.alert.*", - "hm.agent.shutdown.*", - "hm.director.alert" - ] - } - FULL_PERMISSIONS: { - publish: [">"] - subscribe: [">"] - } - - certificate_clients: [ - {client_name: director.bosh-internal, permissions: $DIRECTOR_PERMISSIONS}, - {client_name: agent.bosh-internal, permissions: $AGENT_PERMISSIONS}, - {client_name: bootstrap.agent.bosh-internal, permissions: $AGENT_PERMISSIONS}, - {client_name: hm.bosh-internal, permissions: $HM_PERMISSIONS}, - {client_name: integration.test.bosh-internal, permissions: $FULL_PERMISSIONS}, + users = [ + { + user: "C=USA, O=Cloud Foundry, CN=default.director.bosh-internal" + permissions: { + publish: [ "agent.*", "hm.director.alert" ] + subscribe: [ "director.>" ] + } + }, + { + user: "C=USA, O=Cloud Foundry, CN=default.hm.bosh-internal" + permissions: { + publish: [] + subscribe: [ + "hm.agent.heartbeat.*", + "hm.agent.alert.*", + "hm.agent.shutdown.*", + "hm.director.alert" + ] + } + }, + { + user: "C=USA, O=Cloud Foundry, CN=default.integration.test.bosh-internal" + permissions: { + publish: [">"] + subscribe: [">"] + } + } ] timeout: 5 @@ -53,7 +42,12 @@ tls { cert_file: "<%= nats_certificate_paths['server']['certificate_path'] %>" key_file: "<%= nats_certificate_paths['server']['private_key_path'] %>" ca_file: "<%= nats_certificate_paths['ca_path'] %>" - verify: true + verify_and_map: true timeout: 5 - enable_cert_authorization: true } + +ping_interval: 5s +ping_max: 2 +max_payload: 1048576 + +include ./auth.json diff --git a/src/spec/assets/uaa_config/asymmetric/uaa.yml b/src/spec/assets/uaa_config/asymmetric/uaa.yml index 95d36584f3f..70e7ae40140 100644 --- a/src/spec/assets/uaa_config/asymmetric/uaa.yml +++ b/src/spec/assets/uaa_config/asymmetric/uaa.yml @@ -106,6 +106,14 @@ uaa: authorities: uaa.none,bosh.admin secret: "secret" redirect-uri: http://127.0.0.1 + nats-sync: + id: nats-sync + override: true + authorized-grant-types: client_credentials + scope: '' + authorities: bosh.admin + secret: "secret" + redirect-uri: http://127.0.0.1 production_team: id: production_team override: true diff --git a/src/spec/integration_support/artifact_installer.rb b/src/spec/integration_support/artifact_installer.rb deleted file mode 100644 index 311d497a006..00000000000 --- a/src/spec/integration_support/artifact_installer.rb +++ /dev/null @@ -1,70 +0,0 @@ -module IntegrationSupport - - class ArtifactInstaller - INSTALL_BINARY_SCRIPT = File.expand_path(File.join(File.dirname(__FILE__), 'artifact_installer_install_binary.sh')) - attr_reader :info, :install_dir, :executable_name - - def initialize(install_dir, executable_name, options) - @install_dir = install_dir - @executable_name = executable_name - @info = Info.new(executable_name, - options[:version], - options[:darwin_sha256], - options[:linux_sha256], - options[:bucket_name]) - end - - def install - FileUtils.mkdir_p(install_dir) - - downloaded_file_path = download - - FileUtils.copy(downloaded_file_path, executable_path) - FileUtils.remove(downloaded_file_path, :force => true) - File.chmod(0700, executable_path) - end - - def executable_path - File.join(IntegrationSupport::Constants::BOSH_REPO_SRC_DIR, install_dir, executable_name) - end - - private - - class Info < Struct.new(:name, :rev, :darwin_sha256, :linux_sha256, :bucket_name) - def sha256 - darwin? ? darwin_sha256 : linux_sha256 - end - - def platform - darwin? ? 'darwin' : 'linux' - end - - def file_name_to_download - "#{name}-#{rev}-#{platform}-amd64" - end - - private - - def darwin? - RUBY_PLATFORM =~ /darwin/ - end - end - - def download - destination_path = File.join(install_dir, info.file_name_to_download) - - unless File.exist?(destination_path) - retryable.retryer do - `#{INSTALL_BINARY_SCRIPT} #{info.file_name_to_download} #{destination_path} #{info.sha256} #{info.bucket_name}` - $? == 0 - end - end - - destination_path - end - - def retryable - Bosh::Retryable.new({tries: 6}) - end - end -end diff --git a/src/spec/integration_support/artifact_installer_install_binary.sh b/src/spec/integration_support/artifact_installer_install_binary.sh deleted file mode 100755 index f2319c44a02..00000000000 --- a/src/spec/integration_support/artifact_installer_install_binary.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/usr/bin/env bash -set -eu -set -x - -VERSIONED_FILENAME=$1 -DESTINATION_PATH=$2 -ARCHIVE_SHA=$3 -BUCKET_FOLDER=$4 - -ARCHIVE_URL="https://s3.amazonaws.com/${BUCKET_FOLDER}/${VERSIONED_FILENAME}" - -wget -q -c "${ARCHIVE_URL}" -O "${DESTINATION_PATH}" -if [ -n "${ARCHIVE_SHA}" ]; then - echo "${ARCHIVE_SHA} ${DESTINATION_PATH}" | shasum -a 256 -c - -fi diff --git a/src/spec/integration_support/gnatsd_manager.rb b/src/spec/integration_support/gnatsd_manager.rb index d8b11bdfd33..26e33aced25 100644 --- a/src/spec/integration_support/gnatsd_manager.rb +++ b/src/spec/integration_support/gnatsd_manager.rb @@ -1,5 +1,3 @@ -require 'integration_support/artifact_installer' - module IntegrationSupport class GnatsdManager def self.install @@ -11,19 +9,43 @@ def self.executable_path end def self.installer - @installer ||= - ArtifactInstaller.new( - File.join('tmp', 'gnatsd'), - 'gnatsd', - { - version: '1.3.0-bosh.10', - darwin_sha256: 'fac87b6b9b46830551f32f22930a61e2162edf025304f0f2ce7282b4350003f7', - linux_sha256: 'e5362a7c88ed92d4f4263b1b725e901fe29da220c3548e37570793776b5f6d51', - bucket_name: 'bosh-gnatsd', - } - ) + @installer ||= NatsServerBlobInstaller.new end private_class_method :installer end + + class NatsServerBlobInstaller + INSTALL_DIR = File.join(IntegrationSupport::Constants::BOSH_REPO_SRC_DIR, 'tmp', 'integration-nats') + + def install + Dir.chdir(IntegrationSupport::Constants::BOSH_REPO_ROOT) do + run_command("mkdir -p #{INSTALL_DIR}") + run_command('bosh sync-blobs') + run_command('tar -zxvf blobs/nats/nats-server-*.tar.gz -C /tmp') + run_command("cp /tmp/nats-server-*/nats-server #{executable_path}") + run_command("chmod +x #{executable_path}") + end + end + + def executable_path + File.join(INSTALL_DIR, 'nats-server') + end + + private + + def run_command(command, environment = {}) + io = IO.popen([environment, 'bash', '-c', command]) + + lines = + io.each_with_object("") do |line, collect| + collect << line + puts line.chomp + end + + io.close + + lines + end + end end diff --git a/src/spec/integration_support/sandbox.rb b/src/spec/integration_support/sandbox.rb index 2e894ea9ef1..63136f10809 100644 --- a/src/spec/integration_support/sandbox.rb +++ b/src/spec/integration_support/sandbox.rb @@ -25,8 +25,12 @@ class Sandbox HM_CONFIG = 'health_monitor.yml' DEFAULT_HM_CONF_TEMPLATE_NAME = 'health_monitor.yml.erb' + NATS_SERVER_PID = 'nats.pid' NATS_CONFIG = 'nats.conf' + NATS_SYNC_CONFIG = 'bosh_nats_sync_config.yml' + NATS_SYNC_AUTH_JSON = 'auth.json' DEFAULT_NATS_CONF_TEMPLATE_NAME = 'nats.conf.erb' + DEFAULT_NATS_SYNC_CONF_TEMPLATE_NAME = 'bosh_nats_sync_config.yml.erb' DIRECTOR_CERTIFICATE_EXPIRY_JSON_CONFIG = 'director_certificate_expiry.json'.freeze DIRECTOR_CERTIFICATE_EXPIRY_JSON_TEMPLATE_NAME = 'director_certificate_expiry.json.erb'.freeze @@ -53,10 +57,12 @@ class Sandbox attr_reader :cpi attr_reader :nats_log_path + attr_reader :nats_sync_log_path attr_reader :nats_url attr_reader :dummy_cpi_api_version + attr_reader :user_authentication attr_accessor :trusted_certs @@ -95,6 +101,7 @@ def initialize(db_opts, debug, test_env_number) @dummy_cpi_api_version = nil @nats_log_path = File.join(@logs_path, 'nats.log') + @nats_sync_log_path = File.join(@logs_path, 'nats-sync.log') setup_nats @config_server_service = ConfigServerService.new(@port_provider, base_log_path, @logger, test_env_number) @@ -168,8 +175,7 @@ def start @nginx_service.start - @nats_process.start - @nats_socket_connector.try_to_connect + start_nats @db_helper.create_db @@ -256,6 +262,7 @@ def stop @nginx_service.stop @nats_process.stop + @nats_sync_process.stop @health_monitor_process.stop @@ -394,17 +401,23 @@ def director_nats_config def stop_nats @nats_process.stop + @nats_sync_process.stop end def start_nats return if @nats_process.running? nats_template_path = File.join(IntegrationSupport::Constants::SANDBOX_ASSETS_DIR, DEFAULT_NATS_CONF_TEMPLATE_NAME) + nats_sync_template_path = File.join(IntegrationSupport::Constants::SANDBOX_ASSETS_DIR, DEFAULT_NATS_SYNC_CONF_TEMPLATE_NAME) write_in_sandbox(NATS_CONFIG, load_config_template(nats_template_path)) + write_in_sandbox(NATS_SYNC_CONFIG, load_config_template(nats_sync_template_path)) + write_in_sandbox(NATS_SYNC_AUTH_JSON, JSON.dump({})) write_in_sandbox(EXTERNAL_CPI_CONFIG, load_config_template(EXTERNAL_CPI_CONFIG_TEMPLATE)) setup_nats @nats_process.start @nats_socket_connector.try_to_connect + write_in_sandbox(NATS_SERVER_PID, @nats_process.pid) + @nats_sync_process.start end private @@ -507,11 +520,18 @@ def base_log_path end def setup_nats - gnatsd_path = IntegrationSupport::GnatsdManager.executable_path - conf = File.join(sandbox_root, NATS_CONFIG) + nats_server_conf_path = File.join(sandbox_root, NATS_CONFIG) @nats_process = Service.new( - %W[#{gnatsd_path} -c #{conf} -T -D ], + %W[#{nats_server_executable_path} -c #{nats_server_conf_path} -T -D ], + {stdout: $stdout, stderr: $stderr}, + @logger + ) + + nats_sync_path = File.join(IntegrationSupport::Constants::BOSH_REPO_SRC_DIR, 'bosh-nats-sync', 'bin', 'bosh-nats-sync') + nats_sync_conf_path = File.join(sandbox_root, 'bosh_nats_sync_config.yml') + @nats_sync_process = Service.new( + %W[#{nats_sync_path} -c #{nats_sync_conf_path}], {stdout: $stdout, stderr: $stderr}, @logger ) @@ -539,6 +559,30 @@ def get_nats_client_ca_private_key_path File.join(IntegrationSupport::Constants::SANDBOX_ASSETS_DIR, 'nats_server', 'certs', 'rootCA.key') end + def nats_sync_director_subject_file_path + File.join(IntegrationSupport::Constants::BOSH_REPO_SRC_DIR, 'bosh-nats-sync', 'spec', 'assets', 'director-subject') + end + + def nats_sync_hm_subject_file_path + File.join(IntegrationSupport::Constants::BOSH_REPO_SRC_DIR, 'bosh-nats-sync', 'spec', 'assets', 'hm-subject') + end + + def nats_sync_auth_json_path + File.join(sandbox_root, NATS_SYNC_AUTH_JSON) + end + + def nats_server_executable_path + IntegrationSupport::GnatsdManager.executable_path + end + + def nats_server_pid_path + File.join(sandbox_root, NATS_SERVER_PID) + end + + def uaa_ca_cert_path + IntegrationSupport::UaaService::SERVER_CERT + end + attr_reader :director_tmp_path, :task_logs_dir end end