Add LocalClient for filesystem-based blobstore

config/cloud_controller.yml

@@ -175,36 +175,32 @@ resource_pool:
   maximum_size: 42
   minimum_size: 1
   resource_directory_key: "spec-cc-resources"
-  fog_connection:
-    blobstore_timeout: 5
-    provider: "Local"
+  blobstore_type: local-temp-storage
+  fog_connection: {}
   fog_aws_storage_options: {}
   fog_gcp_storage_options: {}
 
 packages:
   app_package_directory_key: "cc-packages"
   max_package_size: 42
   max_valid_packages_stored: 42
-  fog_connection:
-    blobstore_timeout: 5
-    provider: "Local"
+  blobstore_type: local-temp-storage
+  fog_connection: {}
   fog_aws_storage_options: {}
   fog_gcp_storage_options: {}
 
 droplets:
   droplet_directory_key: cc-droplets
   max_staged_droplets_stored: 42
-  fog_connection:
-    blobstore_timeout: 5
-    provider: "Local"
+  blobstore_type: local-temp-storage
+  fog_connection: {}
   fog_aws_storage_options: {}
   fog_gcp_storage_options: {}
 
 buildpacks:
   buildpack_directory_key: cc-buildpacks
-  fog_connection:
-    blobstore_timeout: 5
-    provider: "Local"
+  blobstore_type: local-temp-storage
+  fog_connection: {}
   fog_aws_storage_options: {}
   fog_gcp_storage_options: {}
 

lib/cloud_controller/blobstore/client_provider.rb

@@ -3,6 +3,7 @@
 require 'cloud_controller/blobstore/fog/fog_client'
 require 'cloud_controller/blobstore/error_handling_client'
 require 'cloud_controller/blobstore/webdav/dav_client'
+require 'cloud_controller/blobstore/local/local_client'
 require 'cloud_controller/blobstore/safe_delete_client'
 require 'cloud_controller/blobstore/storage_cli/storage_cli_client'
 require 'google/apis/errors'
@@ -11,11 +12,15 @@ module CloudController
   module Blobstore
     class ClientProvider
       def self.provide(options:, directory_key:, root_dir: nil, resource_type: nil)
-        if options[:blobstore_type].blank? || (options[:blobstore_type] == 'fog')
-          provide_fog(options, directory_key, root_dir)
-        elsif options[:blobstore_type] == 'storage-cli'
-          # storage-cli is an experimental feature and not yet fully implemented. !!! DO NOT USE IN PRODUCTION !!!
+        case options[:blobstore_type]
+        when 'local'
+          provide_local(options, directory_key, root_dir, use_temp_storage: false)
+        when 'local-temp-storage'
+          provide_local(options, directory_key, root_dir, use_temp_storage: true)
+        when 'storage-cli'
           provide_storage_cli(options, directory_key, root_dir, resource_type)
+        when 'fog', nil, ''
+          provide_fog(options, directory_key, root_dir)
         else
           provide_webdav(options, directory_key, root_dir)
         end
@@ -54,6 +59,23 @@ def provide_fog(options, directory_key, root_dir)
           Client.new(ErrorHandlingClient.new(SafeDeleteClient.new(retryable_client, root_dir)))
         end
 
+        def provide_local(options, directory_key, root_dir, use_temp_storage:)
+          client = LocalClient.new(
+            directory_key: directory_key,
+            base_path: options[:local_blobstore_path],
+            root_dir: root_dir,
+            min_size: options[:minimum_size],
+            max_size: options[:maximum_size],
+            use_temp_storage: use_temp_storage
+          )
+
+          logger = Steno.logger('cc.blobstore.local_client')
+          errors = [StandardError]
+          retryable_client = RetryableClient.new(client:, errors:, logger:)
+
+          Client.new(SafeDeleteClient.new(retryable_client, root_dir))
+        end
+
         def provide_webdav(options, directory_key, root_dir)
           client = DavClient.build(
             options.fetch(:webdav_config),

lib/cloud_controller/blobstore/local/local_blob.rb

@@ -0,0 +1,43 @@
+require 'cloud_controller/blobstore/blob'
+require 'openssl'
+
+module CloudController
+  module Blobstore
+    class LocalBlob < Blob
+      attr_reader :key
+
+      def initialize(key:, file_path:)
+        @key       = key
+        @file_path = file_path
+      end
+
+      def internal_download_url
+        nil
+      end
+
+      def public_download_url
+        nil
+      end
+
+      def local_path
+        @file_path
+      end
+
+      def attributes(*keys)
+        @attributes ||= begin
+          stat = File.stat(@file_path)
+          {
+            etag: OpenSSL::Digest::MD5.file(@file_path).hexdigest,
+            last_modified: stat.mtime.httpdate,
+            content_length: stat.size.to_s,
+            created_at: stat.ctime
+          }
+        end
+
+        return @attributes if keys.empty?
+
+        @attributes.slice(*keys)
+      end
+    end
+  end
+end

lib/cloud_controller/blobstore/local/local_client.rb

@@ -0,0 +1,177 @@
+require 'cloud_controller/blobstore/base_client'
+require 'cloud_controller/blobstore/errors'
+require 'cloud_controller/blobstore/local/local_blob'
+require 'fileutils'
+require 'digest'
+
+module CloudController
+  module Blobstore
+    class LocalClient < BaseClient
+      attr_reader :root_dir
+
+      def initialize(
+        directory_key:,
+        base_path:,
+        root_dir: nil,
+        min_size: nil,
+        max_size: nil,
+        use_temp_storage: false
+      )
+        @directory_key = directory_key
+        @use_temp_storage = use_temp_storage
+        @root_dir      = root_dir
+        @min_size      = min_size || 0
+        @max_size      = max_size
+
+        setup_storage_path(base_path)
+      end
+
+      def local?
+        true
+      end
+
+      def exists?(key)
+        File.exist?(file_path(key))
+      end
+
+      def download_from_blobstore(source_key, destination_path, mode: nil)
+        FileUtils.mkdir_p(File.dirname(destination_path))
+        FileUtils.cp(file_path(source_key), destination_path)
+        File.chmod(mode, destination_path) if mode
+      rescue Errno::ENOENT
+        raise FileNotFound.new("Could not find object '#{source_key}'")
+      end
+
+      def cp_to_blobstore(source_path, destination_key)
+        start     = Time.now.utc
+        log_entry = 'blobstore.cp-skip'
+
+        logger.info('blobstore.cp-start', destination_key: destination_key, source_path: source_path, bucket: @directory_key)
+
+        size = File.size(source_path)
+        if within_limits?(size)
+          destination = file_path(destination_key)
+          FileUtils.mkdir_p(File.dirname(destination))
+          FileUtils.cp(source_path, destination)
+          log_entry = 'blobstore.cp-finish'
+        end
+
+        duration = Time.now.utc - start
+        logger.info(log_entry, destination_key: destination_key, duration_seconds: duration, size: size)
+      rescue Errno::ENOENT => e
+        raise FileNotFound.new("Could not find source file '#{source_path}': #{e.message}")
+      end
+
+      def cp_file_between_keys(source_key, destination_key)
+        source      = file_path(source_key)
+        destination = file_path(destination_key)
+
+        raise FileNotFound.new("Could not find object '#{source_key}'") unless File.exist?(source)
+
+        FileUtils.mkdir_p(File.dirname(destination))
+        FileUtils.cp(source, destination)
+      end
+
+      def delete(key)
+        path = file_path(key)
+        FileUtils.rm_f(path)
+        cleanup_empty_parent_directories(path)
+      end
+
+      def blob(key)
+        path = file_path(key)
+        return unless File.exist?(path)
+
+        LocalBlob.new(key: partitioned_key(key), file_path: path)
+      end
+
+      def delete_blob(blob)
+        path = File.join(@base_path, blob.key)
+        FileUtils.rm_f(path)
+        cleanup_empty_parent_directories(path)
+      end
+
+      def delete_all(_=nil)
+        FileUtils.rm_rf(@base_path)
+        FileUtils.mkdir_p(@base_path)
+      end
+
+      def delete_all_in_path(path)
+        dir = File.join(@base_path, path)
+        FileUtils.rm_rf(dir) if File.directory?(dir)
+      end
+
+      def files_for(prefix, _ignored_directory_prefixes=[])
+        pattern = File.join(@base_path, prefix, '**', '*')
+        Enumerator.new do |yielder|
+          Dir.glob(pattern).each do |file_path|
+            next unless File.file?(file_path)
+
+            relative_path = file_path.sub("#{@base_path}/", '')
+            yielder << LocalBlob.new(key: relative_path, file_path: file_path)
+          end
+        end
+      end
+
+      def ensure_bucket_exists
+        FileUtils.mkdir_p(@base_path)
+      end
+
+      private
+
+      def setup_storage_path(base_path)
+        if use_temp_storage?
+          @base_path = Dir.mktmpdir(['cc-blobstore-', "-#{@directory_key}"])
+          logger.info('storage-mode', mode: 'temp', directory_key: @directory_key, path: @base_path)
+          register_cleanup_hook
+        else
+          raise ArgumentError.new('local_blobstore_path is required for persistent storage') if base_path.nil?
+
+          @base_path = File.join(base_path, @directory_key)
+          FileUtils.mkdir_p(@base_path)
+          logger.info('storage-mode', mode: 'persistent', directory_key: @directory_key, path: @base_path)
+        end
+      end
+
+      def file_path(key)
+        File.join(@base_path, partitioned_key(key))
+      end
+
+      def use_temp_storage?
+        @use_temp_storage
+      end
+
+      def register_cleanup_hook
+        # Register cleanup handler for temp storage mode
+        at_exit do
+          cleanup_temp_storage
+        end
+      end
+
+      def cleanup_temp_storage
+        return unless use_temp_storage? && @base_path && File.directory?(@base_path)
+
+        logger.info('temp-storage-cleanup', directory_key: @directory_key, path: @base_path)
+        FileUtils.rm_rf(@base_path)
+      rescue StandardError => e
+        logger.error('temp-storage-cleanup-failed', error: e.message, path: @base_path)
+      end
+
+      def logger
+        @logger ||= Steno.logger('cc.blobstore.local_client')
+      end
+
+      def cleanup_empty_parent_directories(path)
+        dir = File.dirname(path)
+        # Walk up the directory tree, removing empty directories until we hit the base path
+        while dir != @base_path && dir.start_with?(@base_path)
+          break unless File.directory?(dir)
+          break unless Dir.empty?(dir)
+
+          FileUtils.rmdir(dir)
+          dir = File.dirname(dir)
+        end
+      end
+    end
+  end
+end

lib/cloud_controller/config_schemas/api_schema.rb

@@ -211,37 +211,53 @@ class ApiSchema < VCAP::Config
             maximum_size: Integer,
             minimum_size: Integer,
             resource_directory_key: String,
+            optional(:blobstore_type) => String,
+            optional(:local_blobstore_path) => String,
             fog_connection: Hash,
             optional(:connection_config) => Hash,
             fog_aws_storage_options: Hash,
-            fog_gcp_storage_options: Hash
+            fog_gcp_storage_options: Hash,
+            optional(:webdav_config) => Hash,
+            optional(:cdn) => Hash
           },
 
           buildpacks: {
             buildpack_directory_key: String,
+            optional(:blobstore_type) => String,
+            optional(:local_blobstore_path) => String,
             fog_connection: Hash,
             optional(:connection_config) => Hash,
             fog_aws_storage_options: Hash,
-            fog_gcp_storage_options: Hash
+            fog_gcp_storage_options: Hash,
+            optional(:webdav_config) => Hash,
+            optional(:cdn) => Hash
           },
 
           packages: {
             max_package_size: Integer,
             max_valid_packages_stored: Integer,
             app_package_directory_key: String,
+            optional(:blobstore_type) => String,
+            optional(:local_blobstore_path) => String,
             fog_connection: Hash,
             optional(:connection_config) => Hash,
             fog_aws_storage_options: Hash,
-            fog_gcp_storage_options: Hash
+            fog_gcp_storage_options: Hash,
+            optional(:webdav_config) => Hash,
+            optional(:cdn) => Hash
           },
 
           droplets: {
             droplet_directory_key: String,
             max_staged_droplets_stored: Integer,
+            optional(:blobstore_type) => String,
+            optional(:local_blobstore_path) => String,
             fog_connection: Hash,
             optional(:connection_config) => Hash,
             fog_aws_storage_options: Hash,
-            fog_gcp_storage_options: Hash
+            fog_gcp_storage_options: Hash,
+            optional(:webdav_config) => Hash,
+            optional(:cdn) => Hash
           },
 
           db_encryption_key: enum(String, NilClass),

lib/cloud_controller/config_schemas/blobstore_benchmarks_schema.rb

@@ -9,8 +9,11 @@ class BlobstoreBenchmarksSchema < VCAP::Config
           blobstore_type: String,
           optional(:blobstore_provider) => String,
 
+          optional(:local_blobstore_path) => String,
           optional(:connection_config) => Hash,
           optional(:fog_connection) => Hash,
+          optional(:webdav_config) => Hash,
+          optional(:cdn) => Hash,
 
           fog_aws_storage_options: Hash,
           fog_gcp_storage_options: Hash,