diff --git a/config/cloud_controller.yml b/config/cloud_controller.yml index bdb1f9108f..1395d928f3 100644 --- a/config/cloud_controller.yml +++ b/config/cloud_controller.yml @@ -175,9 +175,8 @@ resource_pool: maximum_size: 42 minimum_size: 1 resource_directory_key: "spec-cc-resources" - fog_connection: - blobstore_timeout: 5 - provider: "Local" + blobstore_type: local-temp-storage + fog_connection: {} fog_aws_storage_options: {} fog_gcp_storage_options: {} @@ -185,26 +184,23 @@ packages: app_package_directory_key: "cc-packages" max_package_size: 42 max_valid_packages_stored: 42 - fog_connection: - blobstore_timeout: 5 - provider: "Local" + blobstore_type: local-temp-storage + fog_connection: {} fog_aws_storage_options: {} fog_gcp_storage_options: {} droplets: droplet_directory_key: cc-droplets max_staged_droplets_stored: 42 - fog_connection: - blobstore_timeout: 5 - provider: "Local" + blobstore_type: local-temp-storage + fog_connection: {} fog_aws_storage_options: {} fog_gcp_storage_options: {} buildpacks: buildpack_directory_key: cc-buildpacks - fog_connection: - blobstore_timeout: 5 - provider: "Local" + blobstore_type: local-temp-storage + fog_connection: {} fog_aws_storage_options: {} fog_gcp_storage_options: {} diff --git a/lib/cloud_controller/blobstore/client_provider.rb b/lib/cloud_controller/blobstore/client_provider.rb index f4b9c45aec..677a8a9325 100644 --- a/lib/cloud_controller/blobstore/client_provider.rb +++ b/lib/cloud_controller/blobstore/client_provider.rb @@ -3,6 +3,7 @@ require 'cloud_controller/blobstore/fog/fog_client' require 'cloud_controller/blobstore/error_handling_client' require 'cloud_controller/blobstore/webdav/dav_client' +require 'cloud_controller/blobstore/local/local_client' require 'cloud_controller/blobstore/safe_delete_client' require 'cloud_controller/blobstore/storage_cli/storage_cli_client' require 'google/apis/errors' @@ -11,11 +12,15 @@ module CloudController module Blobstore class ClientProvider def self.provide(options:, directory_key:, root_dir: nil, resource_type: nil) - if options[:blobstore_type].blank? || (options[:blobstore_type] == 'fog') - provide_fog(options, directory_key, root_dir) - elsif options[:blobstore_type] == 'storage-cli' - # storage-cli is an experimental feature and not yet fully implemented. !!! DO NOT USE IN PRODUCTION !!! + case options[:blobstore_type] + when 'local' + provide_local(options, directory_key, root_dir, use_temp_storage: false) + when 'local-temp-storage' + provide_local(options, directory_key, root_dir, use_temp_storage: true) + when 'storage-cli' provide_storage_cli(options, directory_key, root_dir, resource_type) + when 'fog', nil, '' + provide_fog(options, directory_key, root_dir) else provide_webdav(options, directory_key, root_dir) end @@ -54,6 +59,23 @@ def provide_fog(options, directory_key, root_dir) Client.new(ErrorHandlingClient.new(SafeDeleteClient.new(retryable_client, root_dir))) end + def provide_local(options, directory_key, root_dir, use_temp_storage:) + client = LocalClient.new( + directory_key: directory_key, + base_path: options[:local_blobstore_path], + root_dir: root_dir, + min_size: options[:minimum_size], + max_size: options[:maximum_size], + use_temp_storage: use_temp_storage + ) + + logger = Steno.logger('cc.blobstore.local_client') + errors = [StandardError] + retryable_client = RetryableClient.new(client:, errors:, logger:) + + Client.new(SafeDeleteClient.new(retryable_client, root_dir)) + end + def provide_webdav(options, directory_key, root_dir) client = DavClient.build( options.fetch(:webdav_config), diff --git a/lib/cloud_controller/blobstore/local/local_blob.rb b/lib/cloud_controller/blobstore/local/local_blob.rb new file mode 100644 index 0000000000..a4dd7c6b23 --- /dev/null +++ b/lib/cloud_controller/blobstore/local/local_blob.rb @@ -0,0 +1,43 @@ +require 'cloud_controller/blobstore/blob' +require 'openssl' + +module CloudController + module Blobstore + class LocalBlob < Blob + attr_reader :key + + def initialize(key:, file_path:) + @key = key + @file_path = file_path + end + + def internal_download_url + nil + end + + def public_download_url + nil + end + + def local_path + @file_path + end + + def attributes(*keys) + @attributes ||= begin + stat = File.stat(@file_path) + { + etag: OpenSSL::Digest::MD5.file(@file_path).hexdigest, + last_modified: stat.mtime.httpdate, + content_length: stat.size.to_s, + created_at: stat.ctime + } + end + + return @attributes if keys.empty? + + @attributes.slice(*keys) + end + end + end +end diff --git a/lib/cloud_controller/blobstore/local/local_client.rb b/lib/cloud_controller/blobstore/local/local_client.rb new file mode 100644 index 0000000000..028e944a80 --- /dev/null +++ b/lib/cloud_controller/blobstore/local/local_client.rb @@ -0,0 +1,177 @@ +require 'cloud_controller/blobstore/base_client' +require 'cloud_controller/blobstore/errors' +require 'cloud_controller/blobstore/local/local_blob' +require 'fileutils' +require 'digest' + +module CloudController + module Blobstore + class LocalClient < BaseClient + attr_reader :root_dir + + def initialize( + directory_key:, + base_path:, + root_dir: nil, + min_size: nil, + max_size: nil, + use_temp_storage: false + ) + @directory_key = directory_key + @use_temp_storage = use_temp_storage + @root_dir = root_dir + @min_size = min_size || 0 + @max_size = max_size + + setup_storage_path(base_path) + end + + def local? + true + end + + def exists?(key) + File.exist?(file_path(key)) + end + + def download_from_blobstore(source_key, destination_path, mode: nil) + FileUtils.mkdir_p(File.dirname(destination_path)) + FileUtils.cp(file_path(source_key), destination_path) + File.chmod(mode, destination_path) if mode + rescue Errno::ENOENT + raise FileNotFound.new("Could not find object '#{source_key}'") + end + + def cp_to_blobstore(source_path, destination_key) + start = Time.now.utc + log_entry = 'blobstore.cp-skip' + + logger.info('blobstore.cp-start', destination_key: destination_key, source_path: source_path, bucket: @directory_key) + + size = File.size(source_path) + if within_limits?(size) + destination = file_path(destination_key) + FileUtils.mkdir_p(File.dirname(destination)) + FileUtils.cp(source_path, destination) + log_entry = 'blobstore.cp-finish' + end + + duration = Time.now.utc - start + logger.info(log_entry, destination_key: destination_key, duration_seconds: duration, size: size) + rescue Errno::ENOENT => e + raise FileNotFound.new("Could not find source file '#{source_path}': #{e.message}") + end + + def cp_file_between_keys(source_key, destination_key) + source = file_path(source_key) + destination = file_path(destination_key) + + raise FileNotFound.new("Could not find object '#{source_key}'") unless File.exist?(source) + + FileUtils.mkdir_p(File.dirname(destination)) + FileUtils.cp(source, destination) + end + + def delete(key) + path = file_path(key) + FileUtils.rm_f(path) + cleanup_empty_parent_directories(path) + end + + def blob(key) + path = file_path(key) + return unless File.exist?(path) + + LocalBlob.new(key: partitioned_key(key), file_path: path) + end + + def delete_blob(blob) + path = File.join(@base_path, blob.key) + FileUtils.rm_f(path) + cleanup_empty_parent_directories(path) + end + + def delete_all(_=nil) + FileUtils.rm_rf(@base_path) + FileUtils.mkdir_p(@base_path) + end + + def delete_all_in_path(path) + dir = File.join(@base_path, path) + FileUtils.rm_rf(dir) if File.directory?(dir) + end + + def files_for(prefix, _ignored_directory_prefixes=[]) + pattern = File.join(@base_path, prefix, '**', '*') + Enumerator.new do |yielder| + Dir.glob(pattern).each do |file_path| + next unless File.file?(file_path) + + relative_path = file_path.sub("#{@base_path}/", '') + yielder << LocalBlob.new(key: relative_path, file_path: file_path) + end + end + end + + def ensure_bucket_exists + FileUtils.mkdir_p(@base_path) + end + + private + + def setup_storage_path(base_path) + if use_temp_storage? + @base_path = Dir.mktmpdir(['cc-blobstore-', "-#{@directory_key}"]) + logger.info('storage-mode', mode: 'temp', directory_key: @directory_key, path: @base_path) + register_cleanup_hook + else + raise ArgumentError.new('local_blobstore_path is required for persistent storage') if base_path.nil? + + @base_path = File.join(base_path, @directory_key) + FileUtils.mkdir_p(@base_path) + logger.info('storage-mode', mode: 'persistent', directory_key: @directory_key, path: @base_path) + end + end + + def file_path(key) + File.join(@base_path, partitioned_key(key)) + end + + def use_temp_storage? + @use_temp_storage + end + + def register_cleanup_hook + # Register cleanup handler for temp storage mode + at_exit do + cleanup_temp_storage + end + end + + def cleanup_temp_storage + return unless use_temp_storage? && @base_path && File.directory?(@base_path) + + logger.info('temp-storage-cleanup', directory_key: @directory_key, path: @base_path) + FileUtils.rm_rf(@base_path) + rescue StandardError => e + logger.error('temp-storage-cleanup-failed', error: e.message, path: @base_path) + end + + def logger + @logger ||= Steno.logger('cc.blobstore.local_client') + end + + def cleanup_empty_parent_directories(path) + dir = File.dirname(path) + # Walk up the directory tree, removing empty directories until we hit the base path + while dir != @base_path && dir.start_with?(@base_path) + break unless File.directory?(dir) + break unless Dir.empty?(dir) + + FileUtils.rmdir(dir) + dir = File.dirname(dir) + end + end + end + end +end diff --git a/lib/cloud_controller/config_schemas/api_schema.rb b/lib/cloud_controller/config_schemas/api_schema.rb index c9b5db365f..e6ab9e260b 100644 --- a/lib/cloud_controller/config_schemas/api_schema.rb +++ b/lib/cloud_controller/config_schemas/api_schema.rb @@ -211,37 +211,53 @@ class ApiSchema < VCAP::Config maximum_size: Integer, minimum_size: Integer, resource_directory_key: String, + optional(:blobstore_type) => String, + optional(:local_blobstore_path) => String, fog_connection: Hash, optional(:connection_config) => Hash, fog_aws_storage_options: Hash, - fog_gcp_storage_options: Hash + fog_gcp_storage_options: Hash, + optional(:webdav_config) => Hash, + optional(:cdn) => Hash }, buildpacks: { buildpack_directory_key: String, + optional(:blobstore_type) => String, + optional(:local_blobstore_path) => String, fog_connection: Hash, optional(:connection_config) => Hash, fog_aws_storage_options: Hash, - fog_gcp_storage_options: Hash + fog_gcp_storage_options: Hash, + optional(:webdav_config) => Hash, + optional(:cdn) => Hash }, packages: { max_package_size: Integer, max_valid_packages_stored: Integer, app_package_directory_key: String, + optional(:blobstore_type) => String, + optional(:local_blobstore_path) => String, fog_connection: Hash, optional(:connection_config) => Hash, fog_aws_storage_options: Hash, - fog_gcp_storage_options: Hash + fog_gcp_storage_options: Hash, + optional(:webdav_config) => Hash, + optional(:cdn) => Hash }, droplets: { droplet_directory_key: String, max_staged_droplets_stored: Integer, + optional(:blobstore_type) => String, + optional(:local_blobstore_path) => String, fog_connection: Hash, optional(:connection_config) => Hash, fog_aws_storage_options: Hash, - fog_gcp_storage_options: Hash + fog_gcp_storage_options: Hash, + optional(:webdav_config) => Hash, + optional(:cdn) => Hash }, db_encryption_key: enum(String, NilClass), diff --git a/lib/cloud_controller/config_schemas/blobstore_benchmarks_schema.rb b/lib/cloud_controller/config_schemas/blobstore_benchmarks_schema.rb index 34767edeae..82e28c0a71 100644 --- a/lib/cloud_controller/config_schemas/blobstore_benchmarks_schema.rb +++ b/lib/cloud_controller/config_schemas/blobstore_benchmarks_schema.rb @@ -9,8 +9,11 @@ class BlobstoreBenchmarksSchema < VCAP::Config blobstore_type: String, optional(:blobstore_provider) => String, + optional(:local_blobstore_path) => String, optional(:connection_config) => Hash, optional(:fog_connection) => Hash, + optional(:webdav_config) => Hash, + optional(:cdn) => Hash, fog_aws_storage_options: Hash, fog_gcp_storage_options: Hash, diff --git a/lib/cloud_controller/config_schemas/clock_schema.rb b/lib/cloud_controller/config_schemas/clock_schema.rb index a822b57ca3..17a681c482 100644 --- a/lib/cloud_controller/config_schemas/clock_schema.rb +++ b/lib/cloud_controller/config_schemas/clock_schema.rb @@ -120,35 +120,51 @@ class ClockSchema < VCAP::Config maximum_size: Integer, minimum_size: Integer, resource_directory_key: String, + optional(:blobstore_type) => String, + optional(:local_blobstore_path) => String, fog_connection: Hash, optional(:connection_config) => Hash, fog_aws_storage_options: Hash, - fog_gcp_storage_options: Hash + fog_gcp_storage_options: Hash, + optional(:webdav_config) => Hash, + optional(:cdn) => Hash }, buildpacks: { buildpack_directory_key: String, + optional(:blobstore_type) => String, + optional(:local_blobstore_path) => String, fog_connection: Hash, optional(:connection_config) => Hash, fog_aws_storage_options: Hash, - fog_gcp_storage_options: Hash + fog_gcp_storage_options: Hash, + optional(:webdav_config) => Hash, + optional(:cdn) => Hash }, packages: { max_package_size: Integer, app_package_directory_key: String, + optional(:blobstore_type) => String, + optional(:local_blobstore_path) => String, fog_connection: Hash, optional(:connection_config) => Hash, fog_aws_storage_options: Hash, - fog_gcp_storage_options: Hash + fog_gcp_storage_options: Hash, + optional(:webdav_config) => Hash, + optional(:cdn) => Hash }, droplets: { droplet_directory_key: String, + optional(:blobstore_type) => String, + optional(:local_blobstore_path) => String, fog_connection: Hash, optional(:connection_config) => Hash, fog_aws_storage_options: Hash, - fog_gcp_storage_options: Hash + fog_gcp_storage_options: Hash, + optional(:webdav_config) => Hash, + optional(:cdn) => Hash }, db_encryption_key: enum(String, NilClass), diff --git a/lib/cloud_controller/config_schemas/deployment_updater_schema.rb b/lib/cloud_controller/config_schemas/deployment_updater_schema.rb index aed8877c8c..b31d76449b 100644 --- a/lib/cloud_controller/config_schemas/deployment_updater_schema.rb +++ b/lib/cloud_controller/config_schemas/deployment_updater_schema.rb @@ -122,35 +122,51 @@ class DeploymentUpdaterSchema < VCAP::Config maximum_size: Integer, minimum_size: Integer, resource_directory_key: String, + optional(:blobstore_type) => String, + optional(:local_blobstore_path) => String, fog_connection: Hash, optional(:connection_config) => Hash, fog_aws_storage_options: Hash, - fog_gcp_storage_options: Hash + fog_gcp_storage_options: Hash, + optional(:webdav_config) => Hash, + optional(:cdn) => Hash }, buildpacks: { buildpack_directory_key: String, + optional(:blobstore_type) => String, + optional(:local_blobstore_path) => String, fog_connection: Hash, optional(:connection_config) => Hash, fog_aws_storage_options: Hash, - fog_gcp_storage_options: Hash + fog_gcp_storage_options: Hash, + optional(:webdav_config) => Hash, + optional(:cdn) => Hash }, packages: { max_package_size: Integer, app_package_directory_key: String, + optional(:blobstore_type) => String, + optional(:local_blobstore_path) => String, fog_connection: Hash, optional(:connection_config) => Hash, fog_aws_storage_options: Hash, - fog_gcp_storage_options: Hash + fog_gcp_storage_options: Hash, + optional(:webdav_config) => Hash, + optional(:cdn) => Hash }, droplets: { droplet_directory_key: String, + optional(:blobstore_type) => String, + optional(:local_blobstore_path) => String, fog_connection: Hash, optional(:connection_config) => Hash, fog_aws_storage_options: Hash, - fog_gcp_storage_options: Hash + fog_gcp_storage_options: Hash, + optional(:webdav_config) => Hash, + optional(:cdn) => Hash }, stacks_file: String, diff --git a/lib/cloud_controller/config_schemas/worker_schema.rb b/lib/cloud_controller/config_schemas/worker_schema.rb index 4970d23863..92f1ebad88 100644 --- a/lib/cloud_controller/config_schemas/worker_schema.rb +++ b/lib/cloud_controller/config_schemas/worker_schema.rb @@ -112,36 +112,52 @@ class WorkerSchema < VCAP::Config maximum_size: Integer, minimum_size: Integer, resource_directory_key: String, + optional(:blobstore_type) => String, + optional(:local_blobstore_path) => String, fog_connection: Hash, optional(:connection_config) => Hash, fog_aws_storage_options: Hash, - fog_gcp_storage_options: Hash + fog_gcp_storage_options: Hash, + optional(:webdav_config) => Hash, + optional(:cdn) => Hash }, buildpacks: { buildpack_directory_key: String, + optional(:blobstore_type) => String, + optional(:local_blobstore_path) => String, fog_connection: Hash, optional(:connection_config) => Hash, fog_aws_storage_options: Hash, - fog_gcp_storage_options: Hash + fog_gcp_storage_options: Hash, + optional(:webdav_config) => Hash, + optional(:cdn) => Hash }, packages: { max_package_size: Integer, max_valid_packages_stored: Integer, app_package_directory_key: String, + optional(:blobstore_type) => String, + optional(:local_blobstore_path) => String, fog_connection: Hash, optional(:connection_config) => Hash, fog_aws_storage_options: Hash, - fog_gcp_storage_options: Hash + fog_gcp_storage_options: Hash, + optional(:webdav_config) => Hash, + optional(:cdn) => Hash }, droplets: { droplet_directory_key: String, + optional(:blobstore_type) => String, + optional(:local_blobstore_path) => String, fog_connection: Hash, optional(:connection_config) => Hash, fog_aws_storage_options: Hash, - fog_gcp_storage_options: Hash + fog_gcp_storage_options: Hash, + optional(:webdav_config) => Hash, + optional(:cdn) => Hash }, db_encryption_key: enum(String, NilClass), diff --git a/spec/unit/lib/cloud_controller/blobstore/client_provider_spec.rb b/spec/unit/lib/cloud_controller/blobstore/client_provider_spec.rb index aef7473375..bd0c1685a2 100644 --- a/spec/unit/lib/cloud_controller/blobstore/client_provider_spec.rb +++ b/spec/unit/lib/cloud_controller/blobstore/client_provider_spec.rb @@ -162,6 +162,53 @@ module Blobstore } end end + + context 'when local is requested' do + let(:blobstore_type) { 'local' } + let(:base_path) { Dir.mktmpdir } + + after do + FileUtils.rm_rf(base_path) + end + + before do + options.merge!(local_blobstore_path: base_path, minimum_size: 100, maximum_size: 1000) + end + + it 'provides a local client' do + allow(LocalClient).to receive(:new).and_call_original + ClientProvider.provide(options: options, directory_key: 'key') + expect(LocalClient).to have_received(:new).with( + directory_key: 'key', + base_path: base_path, + root_dir: nil, + min_size: 100, + max_size: 1000, + use_temp_storage: false + ) + end + end + + context 'when local-temp-storage is requested' do + let(:blobstore_type) { 'local-temp-storage' } + + before do + options.merge!(minimum_size: 100, maximum_size: 1000) + end + + it 'provides a local client with temp storage enabled' do + allow(LocalClient).to receive(:new).and_call_original + ClientProvider.provide(options: options, directory_key: 'key') + expect(LocalClient).to have_received(:new).with( + directory_key: 'key', + base_path: nil, + root_dir: nil, + min_size: 100, + max_size: 1000, + use_temp_storage: true + ) + end + end end end end diff --git a/spec/unit/lib/cloud_controller/blobstore/local/local_blob_spec.rb b/spec/unit/lib/cloud_controller/blobstore/local/local_blob_spec.rb new file mode 100644 index 0000000000..7eed651b68 --- /dev/null +++ b/spec/unit/lib/cloud_controller/blobstore/local/local_blob_spec.rb @@ -0,0 +1,80 @@ +require 'spec_helper' + +module CloudController + module Blobstore + RSpec.describe LocalBlob do + let(:key) { 'te/st/test-key' } + let(:file_path) { Tempfile.new('local_blob_test').path } + let(:file_content) { 'test content for blob' } + + subject(:blob) { LocalBlob.new(key: key, file_path: file_path) } + + before do + File.write(file_path, file_content) + end + + after do + FileUtils.rm_f(file_path) + end + + describe '#key' do + it 'returns the key' do + expect(blob.key).to eq('te/st/test-key') + end + end + + describe '#local_path' do + it 'returns the file path' do + expect(blob.local_path).to eq(file_path) + end + end + + describe '#internal_download_url' do + it 'returns nil' do + expect(blob.internal_download_url).to be_nil + end + end + + describe '#public_download_url' do + it 'returns nil' do + expect(blob.public_download_url).to be_nil + end + end + + describe '#attributes' do + it 'returns all attributes when no keys specified' do + attrs = blob.attributes + expect(attrs).to include(:etag, :last_modified, :content_length, :created_at) + end + + it 'returns the etag as MD5 hash of file content' do + expected_etag = OpenSSL::Digest::MD5.hexdigest(file_content) + expect(blob.attributes[:etag]).to eq(expected_etag) + end + + it 'returns the content length as string' do + expect(blob.attributes[:content_length]).to eq(file_content.length.to_s) + end + + it 'returns last_modified in httpdate format' do + expect(blob.attributes[:last_modified]).to match(/\w+, \d+ \w+ \d+ \d+:\d+:\d+ GMT/) + end + + it 'returns created_at as a Time object' do + expect(blob.attributes[:created_at]).to be_a(Time) + end + + it 'returns only requested keys when specified' do + attrs = blob.attributes(:etag, :content_length) + expect(attrs.keys).to contain_exactly(:etag, :content_length) + end + + it 'caches attributes' do + first_call = blob.attributes + second_call = blob.attributes + expect(first_call).to be(second_call) + end + end + end + end +end diff --git a/spec/unit/lib/cloud_controller/blobstore/local/local_client_spec.rb b/spec/unit/lib/cloud_controller/blobstore/local/local_client_spec.rb new file mode 100644 index 0000000000..07fbb13949 --- /dev/null +++ b/spec/unit/lib/cloud_controller/blobstore/local/local_client_spec.rb @@ -0,0 +1,394 @@ +require 'spec_helper' +require_relative '../client_shared' + +module CloudController + module Blobstore + RSpec.describe LocalClient do + subject(:client) do + LocalClient.new( + directory_key: directory_key, + base_path: base_path, + root_dir: root_dir, + min_size: min_size, + max_size: max_size + ) + end + + let(:directory_key) { 'test-directory' } + let(:base_path) { Dir.mktmpdir } + let(:root_dir) { nil } + let(:min_size) { nil } + let(:max_size) { nil } + let(:logger) { instance_double(Steno::Logger, error: nil, info: nil) } + + before do + allow(Steno).to receive(:logger).and_return(logger) + end + + after do + FileUtils.rm_rf(base_path) + end + + describe 'conforms to blobstore client interface' do + let(:deletable_blob) { instance_double(LocalBlob, key: 'te/st/test-key') } + + before do + # The shared examples expect certain files to exist + # Create the file that the tests will try to download/copy + key = 'blobstore-client-shared-key' + file_path = File.join(base_path, directory_key, 'bl', 'ob', key) + FileUtils.mkdir_p(File.dirname(file_path)) + File.write(file_path, 'shared test content') + end + + it_behaves_like 'a blobstore client' + end + + describe '#local?' do + it 'returns true' do + expect(client.local?).to be(true) + end + end + + describe '#exists?' do + it 'returns false if the file does not exist' do + expect(client.exists?('non-existent-key')).to be(false) + end + + it 'returns true if the file exists' do + key = 'abcdef123456' + path = File.join(base_path, directory_key, 'ab', 'cd', key) + FileUtils.mkdir_p(File.dirname(path)) + File.write(path, 'test content') + + expect(client.exists?(key)).to be(true) + end + end + + describe '#cp_to_blobstore' do + it 'copies a file to the blobstore with partitioned path' do + source_file = Tempfile.new('source') + source_file.write('test content') + source_file.close + + key = 'abcdef123456' + client.cp_to_blobstore(source_file.path, key) + + expected_path = File.join(base_path, directory_key, 'ab', 'cd', key) + expect(File.exist?(expected_path)).to be(true) + expect(File.read(expected_path)).to eq('test content') + + source_file.unlink + end + + it 'does not copy if file size is below min_size' do + client = LocalClient.new(directory_key: directory_key, base_path: base_path, min_size: 100) + + source_file = Tempfile.new('source') + source_file.write('small') + source_file.close + + key = 'test-key' + client.cp_to_blobstore(source_file.path, key) + + expected_path = File.join(base_path, directory_key, 'te', 'st', key) + expect(File.exist?(expected_path)).to be(false) + + source_file.unlink + end + + it 'raises FileNotFound if source file does not exist' do + expect do + client.cp_to_blobstore('/non/existent/source/file', 'some-key') + end.to raise_error(FileNotFound, /Could not find source file/) + end + end + + describe '#download_from_blobstore' do + it 'downloads a file from the blobstore' do + key = 'abcdef123456' + source_path = File.join(base_path, directory_key, 'ab', 'cd', key) + FileUtils.mkdir_p(File.dirname(source_path)) + File.write(source_path, 'stored content') + + dest_file = Tempfile.new('dest') + dest_file.close + dest_path = dest_file.path + + client.download_from_blobstore(key, dest_path) + + expect(File.read(dest_path)).to eq('stored content') + + dest_file.unlink + end + + it 'raises FileNotFound if the file does not exist' do + dest_path = File.join(Dir.mktmpdir, 'dest') + expect do + client.download_from_blobstore('non-existent', dest_path) + end.to raise_error(FileNotFound) + end + + it 'sets file mode if provided' do + key = 'test-key' + source_path = File.join(base_path, directory_key, 'te', 'st', key) + FileUtils.mkdir_p(File.dirname(source_path)) + File.write(source_path, 'content') + + dest_file = Tempfile.new('dest') + dest_file.close + dest_path = dest_file.path + + client.download_from_blobstore(key, dest_path, mode: 0o600) + + expect(File.stat(dest_path).mode.to_s(8)[-3..]).to eq('600') + + dest_file.unlink + end + end + + describe '#cp_file_between_keys' do + it 'copies a file from one key to another' do + source_key = 'source123456' + dest_key = 'dest123456' + + source_path = File.join(base_path, directory_key, 'so', 'ur', source_key) + FileUtils.mkdir_p(File.dirname(source_path)) + File.write(source_path, 'content to copy') + + client.cp_file_between_keys(source_key, dest_key) + + dest_path = File.join(base_path, directory_key, 'de', 'st', dest_key) + expect(File.exist?(dest_path)).to be(true) + expect(File.read(dest_path)).to eq('content to copy') + end + + it 'raises FileNotFound if source does not exist' do + expect do + client.cp_file_between_keys('non-existent', 'dest') + end.to raise_error(FileNotFound) + end + end + + describe '#delete' do + it 'deletes a file' do + key = 'test123456' + file_path = File.join(base_path, directory_key, 'te', 'st', key) + FileUtils.mkdir_p(File.dirname(file_path)) + File.write(file_path, 'content') + + expect(File.exist?(file_path)).to be(true) + client.delete(key) + expect(File.exist?(file_path)).to be(false) + end + + it 'does not raise error if file does not exist' do + expect { client.delete('non-existent') }.not_to raise_error + end + + it 'cleans up empty parent directories after deletion' do + key = 'test123456' + file_path = File.join(base_path, directory_key, 'te', 'st', key) + FileUtils.mkdir_p(File.dirname(file_path)) + File.write(file_path, 'content') + + client.delete(key) + + # The partitioned directories (te/st/) should be removed since they're empty + expect(Dir.exist?(File.join(base_path, directory_key, 'te', 'st'))).to be(false) + expect(Dir.exist?(File.join(base_path, directory_key, 'te'))).to be(false) + # But the base directory should still exist + expect(Dir.exist?(File.join(base_path, directory_key))).to be(true) + end + + it 'does not remove non-empty parent directories' do + key1 = 'test123456' + key2 = 'test789012' + file_path1 = File.join(base_path, directory_key, 'te', 'st', key1) + file_path2 = File.join(base_path, directory_key, 'te', 'st', key2) + FileUtils.mkdir_p(File.dirname(file_path1)) + File.write(file_path1, 'content1') + File.write(file_path2, 'content2') + + client.delete(key1) + + # The directory should still exist because key2 is still there + expect(Dir.exist?(File.join(base_path, directory_key, 'te', 'st'))).to be(true) + expect(File.exist?(file_path2)).to be(true) + end + end + + describe '#blob' do + it 'returns a LocalBlob for an existing file' do + key = 'test123456' + file_path = File.join(base_path, directory_key, 'te', 'st', key) + FileUtils.mkdir_p(File.dirname(file_path)) + File.write(file_path, 'content') + + blob = client.blob(key) + expect(blob).to be_a(LocalBlob) + expect(blob.key).to eq('te/st/test123456') + end + + it 'returns nil for non-existent file' do + expect(client.blob('non-existent')).to be_nil + end + end + + describe '#delete_all' do + it 'deletes all files in the directory' do + %w[test1 test2 test3].each do |key| + path = File.join(base_path, directory_key, 'te', 'st', key) + FileUtils.mkdir_p(File.dirname(path)) + File.write(path, 'content') + end + + client.delete_all + + expect(Dir.exist?(File.join(base_path, directory_key))).to be(true) + expect(Dir.glob(File.join(base_path, directory_key, '**', '*')).select { |f| File.file?(f) }).to be_empty + end + end + + describe '#delete_all_in_path' do + it 'deletes all files in a specific path' do + path1 = File.join(base_path, directory_key, 'path1', 'file1') + path2 = File.join(base_path, directory_key, 'path2', 'file2') + + FileUtils.mkdir_p(File.dirname(path1)) + FileUtils.mkdir_p(File.dirname(path2)) + File.write(path1, 'content1') + File.write(path2, 'content2') + + client.delete_all_in_path('path1') + + expect(File.exist?(path1)).to be(false) + expect(File.exist?(path2)).to be(true) + end + end + + describe '#files_for' do + it 'returns an enumerator of blobs matching the prefix' do + %w[aa/bb/test1 aa/cc/test2 ab/cd/test3].each do |path| + full_path = File.join(base_path, directory_key, path) + FileUtils.mkdir_p(File.dirname(full_path)) + File.write(full_path, 'content') + end + + blobs = client.files_for('aa').to_a + expect(blobs.length).to eq(2) + expect(blobs).to all(be_a(LocalBlob)) + end + end + + describe '#ensure_bucket_exists' do + it 'creates the base path directory' do + new_base = File.join(Dir.mktmpdir, 'new-base') + client = LocalClient.new(directory_key: 'test', base_path: new_base) + + FileUtils.rm_rf(new_base) + expect(Dir.exist?(File.join(new_base, 'test'))).to be(false) + + client.ensure_bucket_exists + + expect(Dir.exist?(File.join(new_base, 'test'))).to be(true) + + FileUtils.rm_rf(new_base) + end + end + + describe 'temp storage mode' do + let(:temp_storage_client) do + LocalClient.new(directory_key: 'temp-test', base_path: nil, use_temp_storage: true) + end + + after do + # Manually clean up since we can't rely on at_exit in tests + path = temp_storage_client.instance_variable_get(:@base_path) + FileUtils.rm_rf(path) if path && File.directory?(path) + end + + it 'creates a temporary directory' do + path = temp_storage_client.instance_variable_get(:@base_path) + expect(path).to include('cc-blobstore-') + expect(path).to include('temp-test') + expect(File.directory?(path)).to be(true) + end + + it 'stores files in the temporary directory' do + source_file = Tempfile.new('temp-source') + source_file.write('temp content') + source_file.close + + key = 'test123456' + temp_storage_client.cp_to_blobstore(source_file.path, key) + + expect(temp_storage_client.exists?(key)).to be(true) + + source_file.unlink + end + + it 'cleans up the temporary directory' do + path = temp_storage_client.instance_variable_get(:@base_path) + expect(File.directory?(path)).to be(true) + + temp_storage_client.send(:cleanup_temp_storage) + + expect(File.directory?(path)).to be(false) + end + + it 'logs error if cleanup fails' do + path = temp_storage_client.instance_variable_get(:@base_path) + + # Need a real logger instance for this test since at_exit runs outside test lifecycle + real_logger = instance_double(Steno::Logger, info: nil, error: nil) + temp_storage_client.instance_variable_set(:@logger, real_logger) + + allow(FileUtils).to receive(:rm_rf).with(path).and_raise(StandardError.new('permission denied')) + + expect(real_logger).to receive(:error).with('temp-storage-cleanup-failed', error: 'permission denied', path: path) + + temp_storage_client.send(:cleanup_temp_storage) + + # Clean up manually since rm_rf was mocked + allow(FileUtils).to receive(:rm_rf).and_call_original + end + end + + describe 'persistent mode (default)' do + it 'requires base_path for persistent storage' do + expect do + LocalClient.new(directory_key: 'persistent-test', base_path: nil) + end.to raise_error(ArgumentError, /local_blobstore_path is required/) + end + + it 'keeps existing files' do + existing_path = File.join(base_path, 'persistent-test', 'old-file') + FileUtils.mkdir_p(File.dirname(existing_path)) + File.write(existing_path, 'old content') + + expect(File.exist?(existing_path)).to be(true) + + LocalClient.new(directory_key: 'persistent-test', base_path: base_path) + + expect(File.exist?(existing_path)).to be(true) + end + + it 'is the default when use_temp_storage is not specified' do + existing_path = File.join(base_path, 'default-test', 'old-file') + FileUtils.mkdir_p(File.dirname(existing_path)) + File.write(existing_path, 'old content') + + expect(File.exist?(existing_path)).to be(true) + + LocalClient.new( + directory_key: 'default-test', + base_path: base_path + ) + + expect(File.exist?(existing_path)).to be(true) + end + end + end + end +end