Reliability improvements to PostgreSQL plugin and performance improvements to SHIELD auth flow

.claude/settings.local.json

@@ -0,0 +1,8 @@
+{
+  "permissions": {
+    "allow": [
+      "Bash(node:*)",
+      "Bash(python3:*)"
+    ]
+  }
+}

.env

@@ -0,0 +1 @@
+VERSION=9.0.0

.gitignore

@@ -72,3 +72,4 @@ demo/shield-agent/in
 demo/shield-core/in
 demo/shared
 demo/cached
+/.vscode

.vscode/dryrun.log

@@ -1,3 +1,4 @@
+make.exe --dry-run --always-make --keep-going --print-directory
 'make.exe' is not recognized as an internal or external command,
 operable program or batch file.
 

Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.21-bookworm as build
+FROM golang:1.23-bookworm as build
 
 RUN apt-get update \
  && apt-get install -y bzip2 gzip unzip curl openssh-client
@@ -9,6 +9,8 @@ RUN curl -sLo /bin/jq https://github.com/jqlang/jq/releases/download/jq-1.7.1/jq
 ARG VERSION
 COPY / /go/src/github.com/shieldproject/shield/
 RUN cd /go/src/github.com/shieldproject/shield \
+ && go mod tidy \
+ && go mod vendor \
  && make build BUILD_TYPE="build -ldflags='-X main.Version=$VERSION'"
 RUN mkdir -p /dist/bin /dist/plugins \
  && mv /go/src/github.com/shieldproject/shield/shieldd \

Makefile

@@ -50,18 +50,33 @@ help.all: cmd/shield/main.go
 	grep case $< | grep '{''{{' | cut -d\" -f 2 | sort | xargs -n1 -I@ ./shield @ -h > $@
 
 # Building Plugins
+JOBS ?= $(shell nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo 1)
+
 plugin: plugins
 plugins:
-	go $(BUILD_TYPE) -mod vendor ./plugin/dummy
+	@echo "Building dummy plugin..."
+	@go $(BUILD_TYPE) -mod vendor ./plugin/dummy || go $(BUILD_TYPE) ./plugin/dummy
 	@for plugin in $$(cat plugins); do \
-		echo building plugin $$plugin...; \
-		go $(BUILD_TYPE) -mod vendor ./plugin/$$plugin; \
+		echo "building plugin $$plugin..."; \
+		if ! go $(BUILD_TYPE) -mod vendor ./plugin/$$plugin; then \
+			GOFLAGS=-mod=mod go $(BUILD_TYPE) ./plugin/$$plugin; \
+		fi; \
 	done
 
 
 demo: clean shield plugins
-	./demo/build
-	(cd demo && docker-compose up)
+	@if [ -x ./demo/build ]; then \
+		./demo/build; \
+	else \
+		echo "(warning) ./demo/build not found; skipping demo build"; \
+	fi
+	(cd docker/demo && docker compose up)
+
+# Local build stack (core/agent/demo/webdav from local source build)
+demo-local:
+	docker compose -f docker-compose.local.yml up --build
+
+dev-local: demo-local
 
 docs: docs/dev/API.md
 	./bin/mkdocs --version latest --docroot /docs --output tmp/docs --style basic
@@ -85,7 +100,13 @@ fixmes: fixme
 fixme:
 	@grep -rn FIXME * | grep -v vendor/ | grep -v README.md | grep --color FIXME || echo "No FIXMES!  YAY!"
 
-dev:
+# Quick local development mode (UI + API) using docker-compose
+# Usage: make dev
+#   then open http://localhost:9009
+dev: demo
+
+# Keep legacy testdev flow for deeper local test sandbox
+dev-test:
 	./bin/testdev
 
 # Deferred: Naming plugins individually, e.g. make plugin dummy

docker-compose.local.yml

@@ -0,0 +1,101 @@
+version: '3'
+services:
+  vault:
+    image: hashicorp/vault:1.14
+    ports: ['8201:8200']
+    networks: [internal]
+    environment:
+      VAULT_API_ADDR: http://127.0.0.1:8200
+      VAULT_LOCAL_CONFIG: >-
+        {
+          "disable_mlock": 1,
+          "backend": {
+            "file": {
+              "path": "/vault/file"
+            }
+          },
+          "listener": {
+            "tcp": {
+              "address": "0.0.0.0:8201",
+              "tls_disable": 1
+            }
+          },
+          "default_lease_ttl": "168h",
+          "max_lease_ttl": "720h"
+        }
+    cap_add:
+      - IPC_LOCK
+    volumes:
+      - 'vault-data:/vault/file'
+
+  core:
+    build:
+      context: .
+      dockerfile: Dockerfile
+      args:
+        VERSION: local
+    image: shieldproject/shield:local-core
+    ports: ['9009:80']
+    networks: [internal]
+    command: ['/shield/init/core']
+    environment:
+      SHIELD_VAULT_ADDRESS: http://vault:8201
+      SHIELD_API_BIND: core:80
+    volumes:
+      - 'shield-data:/var/shield'
+      - 'shield-etc:/etc/shield'
+
+  webdav:
+    build:
+      context: docker/webdav
+      dockerfile: Dockerfile
+    ports: ['9007:80']
+    networks: [internal]
+    environment:
+      USERNAME: webdav
+      PASSWORD: password
+    volumes:
+      - 'webdav-data:/var/webdav'
+
+  agent:
+    build:
+      context: .
+      dockerfile: Dockerfile
+      args:
+        VERSION: local
+    image: shieldproject/shield:local-agent
+    depends_on:
+      - core
+      - webdav
+    ports: ['5444:5444']
+    networks: [internal]
+    command: ['/shield/init/agent']
+    environment:
+      SHIELD_API: 'http://core:80'
+      SHIELD_AGENT_NAME: 'demo-shield-agent'
+    volumes:
+      - 'shield-data:/var/shield'
+      - 'shield-etc:/etc/shield'
+      - 'demo-data:/www'
+
+  demo:
+    build:
+      context: docker/demo
+      dockerfile: Dockerfile
+    depends_on:
+      - core
+      - webdav
+    ports: ['9008:80']
+    networks: [internal]
+    volumes:
+      - 'demo-data:/www'
+
+networks:
+  internal:
+
+volumes:
+  vault-data:
+  shield-data:
+  shield-etc:
+  webdav-data:
+  demo-data:

plugin/postgres/plugin.go

@@ -6,6 +6,7 @@ import (
 	"os"
 	"os/exec"
 	"regexp"
+	"strings"
 
 	fmt "github.com/jhunt/go-ansi"
 
@@ -111,6 +112,14 @@ func main() {
 				Help:    "The absolute path to the bin/ directory that contains the `psql` command.",
 				Default: "/var/vcap/packages/postgres-9.4/bin",
 			},
+			plugin.Field{
+				Mode:    "target",
+				Name:    "pg_skip_permission_check",
+				Type:    "bool",
+				Title:   "Skip permission validation",
+				Help:    "Skip upfront permission checking. WARNING: Use only if you understand the risks. Restore may fail with confusing errors if privileges are insufficient.",
+				Default: "false",
+			},
 		},
 	}
 
@@ -120,15 +129,16 @@ func main() {
 type PostgresPlugin plugin.PluginInfo
 
 type PostgresConnectionInfo struct {
-	Host        string
-	Port        string
-	User        string
-	Password    string
-	Bin         string
-	ReplicaHost string
-	ReplicaPort string
-	Database    string
-	Options     string
+	Host                string
+	Port                string
+	User                string
+	Password            string
+	Bin                 string
+	ReplicaHost         string
+	ReplicaPort         string
+	Database            string
+	Options             string
+	SkipPermissionCheck bool
 }
 
 func (p PostgresPlugin) Meta() plugin.PluginInfo {
@@ -259,6 +269,15 @@ func (p PostgresPlugin) Restore(endpoint plugin.ShieldEndpoint) error {
 
 	setupEnvironmentVariables(pg)
 
+	// First, check if we have permission issues before starting the restore
+	if !pg.SkipPermissionCheck {
+		if err := checkRestorePermissions(pg); err != nil {
+			return err
+		}
+	} else {
+		plugin.DEBUG("Skipping permission check as requested")
+	}
+
 	cmd := exec.Command(fmt.Sprintf("%s/psql", pg.Bin), "-d", "postgres")
 	plugin.DEBUG("Exec: %s/psql -d postgres", pg.Bin)
 	plugin.DEBUG("Redirecting stdout and stderr to stderr")
@@ -316,6 +335,44 @@ func (p PostgresPlugin) Restore(endpoint plugin.ShieldEndpoint) error {
 	return <-scanErr
 }
 
+// checkRestorePermissions performs upfront permission checks before starting restore
+func checkRestorePermissions(pg *PostgresConnectionInfo) error {
+	plugin.DEBUG("Checking restore permissions...")
+
+	// Create a temporary connection to check permissions
+	// Check if user is superuser or has specific database privileges
+	cmd := exec.Command(fmt.Sprintf("%s/psql", pg.Bin), "-d", "postgres", "-t", "-A", "-c",
+		"SELECT CASE WHEN "+
+			"(SELECT COALESCE(usesuper, false) FROM pg_user WHERE usename = current_user) OR "+
+			"pg_has_role(current_user, 'rds_superuser', 'MEMBER') OR "+
+			"(pg_has_role(current_user, 'pg_database_owner', 'MEMBER') AND has_database_privilege(current_user, 'postgres', 'CREATE')) "+
+			"THEN 'SUFFICIENT' ELSE 'INSUFFICIENT' END;")
+
+	cmd.Env = os.Environ()
+	cmd.Env = append(cmd.Env,
+		fmt.Sprintf("PGUSER=%s", pg.User),
+		fmt.Sprintf("PGPASSWORD=%s", pg.Password),
+		fmt.Sprintf("PGHOST=%s", pg.Host),
+		fmt.Sprintf("PGPORT=%s", pg.Port),
+	)
+
+	output, err := cmd.Output()
+	if err != nil {
+		plugin.DEBUG("Failed to check permissions: %s", err)
+		return fmt.Errorf("postgres: failed to verify user privileges: %s", err)
+	}
+
+	result := strings.TrimSpace(string(output))
+	plugin.DEBUG("Permission check result: '%s'", result)
+
+	if result != "SUFFICIENT" {
+		return fmt.Errorf("postgres: insufficient privileges for restore operation. User '%s' needs superuser privileges or database creation rights to safely restore databases", pg.User)
+	}
+
+	plugin.DEBUG("User has sufficient privileges for restore")
+	return nil
+}
+
 func (p PostgresPlugin) Store(endpoint plugin.ShieldEndpoint) (string, int64, error) {
 	return "", 0, plugin.UNIMPLEMENTED
 }
@@ -392,15 +449,22 @@ func pgConnectionInfo(endpoint plugin.ShieldEndpoint) (*PostgresConnectionInfo,
 	}
 	plugin.DEBUG("PGBINDIR: '%s'", bin)
 
+	skipCheck, err := endpoint.BooleanValueDefault("pg_skip_permission_check", false)
+	if err != nil {
+		return nil, err
+	}
+	plugin.DEBUG("PG_SKIP_PERMISSION_CHECK: %t", skipCheck)
+
 	return &PostgresConnectionInfo{
-		Host:        host,
-		Port:        port,
-		User:        user,
-		Password:    password,
-		ReplicaHost: replicahost,
-		ReplicaPort: replicaport,
-		Bin:         bin,
-		Database:    database,
-		Options:     options,
+		Host:                host,
+		Port:                port,
+		User:                user,
+		Password:            password,
+		ReplicaHost:         replicahost,
+		ReplicaPort:         replicaport,
+		Bin:                 bin,
+		Database:            database,
+		Options:             options,
+		SkipPermissionCheck: skipCheck,
 	}, nil
 }