From 1b21fdba0e9c1e90a247c8aa1232c03f3ab7da34 Mon Sep 17 00:00:00 2001 From: Eric Bezzam Date: Wed, 29 Nov 2023 14:21:13 +0100 Subject: [PATCH 01/11] Add black level assertion. --- lensless/utils/image.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lensless/utils/image.py b/lensless/utils/image.py index edb213fc..6985592d 100644 --- a/lensless/utils/image.py +++ b/lensless/utils/image.py @@ -265,6 +265,9 @@ def bayer2rgb_cc( dtype = np.uint16 else: dtype = np.uint8 + assert ( + black_level < 2**nbits - 1 + ), f"Black level ({black_level}) too high for bit depth ({nbits})." # demosaic Bayer data img = cv2.cvtColor(img, cv2.COLOR_BayerRG2RGB) From 37b1053a1417fef4a34fa01ce9b017bc6e6942f6 Mon Sep 17 00:00:00 2001 From: Eric Bezzam Date: Wed, 29 Nov 2023 15:49:57 +0100 Subject: [PATCH 02/11] Remove deafult to RPI HQ for bayer conversion. --- lensless/utils/image.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/lensless/utils/image.py b/lensless/utils/image.py index 6985592d..5d21318f 100644 --- a/lensless/utils/image.py +++ b/lensless/utils/image.py @@ -10,7 +10,6 @@ import cv2 import scipy.signal import numpy as np -from lensless.hardware.constants import RPI_HQ_CAMERA_CCM_MATRIX, RPI_HQ_CAMERA_BLACK_LEVEL try: import torch @@ -223,8 +222,8 @@ def bayer2rgb_cc( nbits, blue_gain=None, red_gain=None, - black_level=RPI_HQ_CAMERA_BLACK_LEVEL, - ccm=RPI_HQ_CAMERA_CCM_MATRIX, + black_level=0, + ccm=None, nbits_out=None, ): """ @@ -269,18 +268,27 @@ def bayer2rgb_cc( black_level < 2**nbits - 1 ), f"Black level ({black_level}) too high for bit depth ({nbits})." + if ccm is None: + ccm = np.eye(3) + ccm = np.array(ccm).astype(np.float32) + # demosaic Bayer data img = cv2.cvtColor(img, cv2.COLOR_BayerRG2RGB) # correction img = img - black_level + img = np.clip(img, 0, 2**nbits - 1).astype(np.float32) + if red_gain: img[:, :, 0] *= red_gain if blue_gain: img[:, :, 2] *= blue_gain img = img / (2**nbits - 1 - black_level) img[img > 1] = 1 - img = (img.reshape(-1, 3, order="F") @ ccm.T).reshape(img.shape, order="F") + img_re = img.reshape(-1, 3, order="F") + img_re = img_re @ ccm.T + img = img_re.reshape(img.shape, order="F") + # img = (img.reshape(-1, 3, order="F") @ ccm.T).reshape(img.shape, order="F") img[img < 0] = 0 img[img > 1] = 1 return (img * (2**nbits_out - 1)).astype(dtype) From c0ec046ac0a6b7951e89eaad35c4dd85131a8b7f Mon Sep 17 00:00:00 2001 From: Eric Bezzam Date: Wed, 29 Nov 2023 15:50:30 +0100 Subject: [PATCH 03/11] Add ccm and black level to sensor config. --- lensless/hardware/sensor.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/lensless/hardware/sensor.py b/lensless/hardware/sensor.py index 0785204e..18ebff47 100644 --- a/lensless/hardware/sensor.py +++ b/lensless/hardware/sensor.py @@ -60,6 +60,8 @@ class SensorParam: BIT_DEPTH = "bit_depth" MAX_EXPOSURE = "max_exposure" # in seconds MIN_EXPOSURE = "min_exposure" # in seconds + BLACK_LEVEL = "black_level" # normalized to [0,1] + CCM_MATRIX = "ccm_matrix" """ @@ -79,6 +81,15 @@ class SensorParam: SensorParam.BIT_DEPTH: [8, 12], SensorParam.MAX_EXPOSURE: 670.74, SensorParam.MIN_EXPOSURE: 0.02, + # https://www.strollswithmydog.com/open-raspberry-pi-high-quality-camera-raw + SensorParam.BLACK_LEVEL: 256.3 / (2**12 - 1), + SensorParam.CCM_MATRIX: np.array( + [ + [2.0659, -0.93119, -0.13421], + [-0.11615, 1.5593, -0.44314], + [0.073694, -0.4368, 1.3636], + ] + ), }, # Raspberry Pi Global Shutter Camera # https://www.raspberrypi.com/products/raspberry-pi-global-shutter-camera/ From a630faf33e4984b0c52cd47f556c84fd69a1eb27 Mon Sep 17 00:00:00 2001 From: Eric Bezzam Date: Wed, 29 Nov 2023 15:51:07 +0100 Subject: [PATCH 04/11] More intuitive on device capture. --- configs/capture.yaml | 26 +++++++-------- scripts/measure/on_device_capture.py | 49 +++++++++++++++++++--------- 2 files changed, 47 insertions(+), 28 deletions(-) diff --git a/configs/capture.yaml b/configs/capture.yaml index 18b6c030..e723cc48 100644 --- a/configs/capture.yaml +++ b/configs/capture.yaml @@ -1,19 +1,19 @@ sensor: rpi_hq -bayer: True +legacy: True # Legacy image capture software (raspistill) or new one (libcamera) +bayer: True # whether to capture Bayer data, otherwise directly capture RGB (demosaiced) data +nbits_bayer: 16 # output bit depth for Bayer data (16 or 8), only used if legacy=True fn: test exp: 0.5 config_pause: 2 -# {'off': 0, 'auto': 1, 'sunlight': 2, 'cloudy': 3, -# 'shade': 4, 'tungsten': 5, 'fluorescent': 6, -# 'incandescent': 7, 'flash': 8, 'horizon': 9} -sensor_mode: "0" -rgb: False -gray: False +sensor_mode: "0" # {'off': 0, 'auto': 1, 'sunlight': 2, 'cloudy': 3, 'shade': 4, 'tungsten': 5, 'fluorescent': 6, 'incandescent': 7, 'flash': 8, 'horizon': 9} iso: 100 -sixteen: True # whether 16 bits or 8 (from Bayer data) -legacy: True -down: null -res: null -nbits_out: 8 awb_gains: null -# awb_gains: [1.9, 1.2] # red, blue \ No newline at end of file + +# -- only one of below should be set +down: null # downsample from maximum resolution +res: null # desired resolution + +# -- below only used if bayer=True, capture Bayer data but return as RGB or grayscale +rgb: False +gray: False +nbits_out: 8 \ No newline at end of file diff --git a/scripts/measure/on_device_capture.py b/scripts/measure/on_device_capture.py index 00c8c470..a2cb05d5 100644 --- a/scripts/measure/on_device_capture.py +++ b/scripts/measure/on_device_capture.py @@ -57,9 +57,8 @@ def capture(config): sensor = config.sensor - assert sensor in SensorOptions.values(), f"Sensor must be one of {SensorOptions.values()}" - bayer = config.bayer + nbits_bayer = config.nbits_bayer fn = config.fn exp = config.exp config_pause = config.config_pause @@ -67,15 +66,36 @@ def capture(config): rgb = config.rgb gray = config.gray iso = config.iso - sixteen = config.sixteen legacy = config.legacy down = config.down res = config.res nbits_out = config.nbits_out + assert sensor in SensorOptions.values(), f"Sensor must be one of {SensorOptions.values()}" + + assert res is None or down is None, "Cannot specify both res and down" + assert rgb is False or gray is False, "Cannot set both rgb and gray" + + supported_bit_depth = sensor_dict[sensor][SensorParam.BIT_DEPTH] assert ( - nbits_out in sensor_dict[sensor][SensorParam.BIT_DEPTH] - ), f"nbits_out must be one of {sensor_dict[sensor][SensorParam.BIT_DEPTH]} for sensor {sensor}" + nbits_out in supported_bit_depth + ), f"nbits_out must be one of {supported_bit_depth} for sensor {sensor}" + assert nbits_bayer in [8, 16], "nbits_bayer must be 8 or 16" + if nbits_bayer == 16: + # TODO may not work if there are multiple bit depths above 8 + nbits_measured = max(supported_bit_depth) + assert nbits_measured > 8 + else: + nbits_measured = 8 + + if SensorParam.BLACK_LEVEL in sensor_dict[sensor]: + black_level = sensor_dict[sensor][SensorParam.BLACK_LEVEL] * (2**nbits_measured - 1) + else: + black_level = 0 + if SensorParam.CCM_MATRIX in sensor_dict[sensor]: + ccm = sensor_dict[sensor][SensorParam.CCM_MATRIX] + else: + ccm = None # https://www.raspberrypi.com/documentation/accessories/camera.html#hardware-specification sensor_param = sensor_dict[sensor] @@ -214,17 +234,16 @@ def capture(config): camera.capture(stream, "jpeg", bayer=True) # get bayer data - if sixteen: - output = np.sum(stream.array, axis=2).astype(np.uint16) + raw_data = np.sum(stream.array, axis=2) + if nbits_bayer == 16: + output = raw_data.astype(np.uint16) else: - output = (np.sum(stream.array, axis=2) >> 2).astype(np.uint8) + output = raw_data / (2 ** max(supported_bit_depth) - 1) * 255 + output = output.astype(np.uint8) + # output = (np.sum(stream.array, axis=2) >> 2).astype(np.uint8) # returning non-bayer data if rgb or gray: - if sixteen: - n_bits = 12 # assuming Raspberry Pi HQ - else: - n_bits = 8 if config.awb_gains is not None: red_gain = config.awb_gains[0] @@ -232,11 +251,11 @@ def capture(config): output_rgb = bayer2rgb_cc( output, - nbits=n_bits, + nbits=nbits_measured, blue_gain=blue_gain, red_gain=red_gain, - black_level=RPI_HQ_CAMERA_BLACK_LEVEL, - ccm=RPI_HQ_CAMERA_CCM_MATRIX, + black_level=black_level, + ccm=ccm, nbits_out=nbits_out, ) From c7aa6f376a0de76283003270994c543efff2646a Mon Sep 17 00:00:00 2001 From: Eric Bezzam Date: Wed, 29 Nov 2023 17:17:48 +0100 Subject: [PATCH 05/11] Add utility for checking capture config. --- configs/capture.yaml | 3 +- lensless/hardware/utils.py | 59 +++++++++++++++++++++++++++- lensless/utils/image.py | 19 +++++---- scripts/measure/on_device_capture.py | 53 +++---------------------- 4 files changed, 75 insertions(+), 59 deletions(-) diff --git a/configs/capture.yaml b/configs/capture.yaml index e723cc48..6392ebc2 100644 --- a/configs/capture.yaml +++ b/configs/capture.yaml @@ -7,7 +7,8 @@ exp: 0.5 config_pause: 2 sensor_mode: "0" # {'off': 0, 'auto': 1, 'sunlight': 2, 'cloudy': 3, 'shade': 4, 'tungsten': 5, 'fluorescent': 6, 'incandescent': 7, 'flash': 8, 'horizon': 9} iso: 100 -awb_gains: null +# awb_gains: null # to use gains estimte by system +awb_gains: [1.9, 1.2] # -- only one of below should be set down: null # downsample from maximum resolution diff --git a/lensless/hardware/utils.py b/lensless/hardware/utils.py index adadbc1c..c49cce21 100644 --- a/lensless/hardware/utils.py +++ b/lensless/hardware/utils.py @@ -6,7 +6,7 @@ import paramiko from pprint import pprint from paramiko.ssh_exception import AuthenticationException, BadHostKeyException, SSHException -from lensless.hardware.sensor import SensorOptions +from lensless.hardware.sensor import SensorOptions, sensor_dict, SensorParam import cv2 from lensless.utils.image import print_image_info from lensless.utils.io import load_image @@ -17,6 +17,63 @@ logging.getLogger("paramiko").setLevel(logging.WARNING) +def check_capture_config(config): + + sensor = config.sensor + nbits_bayer = config.nbits_bayer + exp = config.exp + rgb = config.rgb + gray = config.gray + legacy = config.legacy + down = config.down + res = config.res + nbits_out = config.nbits_out + awb_gains = config.awb_gains + + assert sensor in SensorOptions.values(), f"Sensor must be one of {SensorOptions.values()}" + + assert res is None or down is None, "Cannot specify both res and down" + if res is not None: + assert len(res) == 2, "res must be a tuple of length 2" + + assert rgb is False or gray is False, "Cannot set both rgb and gray" + + supported_bit_depth = sensor_dict[sensor][SensorParam.BIT_DEPTH] + assert ( + nbits_out in supported_bit_depth + ), f"nbits_out must be one of {supported_bit_depth} for sensor {sensor}" + assert nbits_bayer in [8, 16], "nbits_bayer must be 8 or 16" + if nbits_bayer == 16: + # TODO may not work if there are multiple bit depths above 8 + nbits_measured = max(supported_bit_depth) + assert nbits_measured > 8 + else: + nbits_measured = 8 + + if SensorParam.BLACK_LEVEL in sensor_dict[sensor]: + black_level = sensor_dict[sensor][SensorParam.BLACK_LEVEL] * (2**nbits_measured - 1) + else: + black_level = 0 + if SensorParam.CCM_MATRIX in sensor_dict[sensor]: + ccm = sensor_dict[sensor][SensorParam.CCM_MATRIX] + else: + ccm = None + + # https://www.raspberrypi.com/documentation/accessories/camera.html#hardware-specification + sensor_param = sensor_dict[sensor] + assert exp <= sensor_param[SensorParam.MAX_EXPOSURE] + assert exp >= sensor_param[SensorParam.MIN_EXPOSURE] + + if awb_gains is not None: + assert len(awb_gains) == 2, "awb_gains must be a tuple of length 2" + + if sensor == SensorOptions.RPI_GS.value: + assert not legacy, "Legacy capture software not supported for Global Shutter sensor" + + # return sensor parameters + return black_level, ccm, supported_bit_depth, nbits_measured + + def capture( rpi_username, rpi_hostname, diff --git a/lensless/utils/image.py b/lensless/utils/image.py index 5d21318f..e9be1a57 100644 --- a/lensless/utils/image.py +++ b/lensless/utils/image.py @@ -270,25 +270,24 @@ def bayer2rgb_cc( if ccm is None: ccm = np.eye(3) - ccm = np.array(ccm).astype(np.float32) + ccm = ccm.copy().astype(np.float32) # demosaic Bayer data - img = cv2.cvtColor(img, cv2.COLOR_BayerRG2RGB) + img = cv2.cvtColor(img, cv2.COLOR_BayerRG2RGB).astype(np.float32) # correction - img = img - black_level - img = np.clip(img, 0, 2**nbits - 1).astype(np.float32) - + img -= black_level if red_gain: img[:, :, 0] *= red_gain if blue_gain: img[:, :, 2] *= blue_gain - img = img / (2**nbits - 1 - black_level) + np.clip(img, 0, 2**nbits - 1, out=img) + img /= 2**nbits - 1 - black_level img[img > 1] = 1 - img_re = img.reshape(-1, 3, order="F") - img_re = img_re @ ccm.T - img = img_re.reshape(img.shape, order="F") - # img = (img.reshape(-1, 3, order="F") @ ccm.T).reshape(img.shape, order="F") + # img_re = img.reshape(-1, 3, order="F") + # img_re = img_re @ ccm.T + # img = img_re.reshape(img.shape, order="F") + img = (img.reshape(-1, 3, order="F") @ ccm.T).reshape(img.shape, order="F") img[img < 0] = 0 img[img > 1] = 1 return (img * (2**nbits_out - 1)).astype(dtype) diff --git a/scripts/measure/on_device_capture.py b/scripts/measure/on_device_capture.py index a2cb05d5..8196a79f 100644 --- a/scripts/measure/on_device_capture.py +++ b/scripts/measure/on_device_capture.py @@ -31,9 +31,8 @@ import numpy as np from time import sleep from PIL import Image -from lensless.hardware.utils import get_distro +from lensless.hardware.utils import get_distro, check_capture_config from lensless.utils.image import bayer2rgb_cc, rgb2gray, resize -from lensless.hardware.constants import RPI_HQ_CAMERA_CCM_MATRIX, RPI_HQ_CAMERA_BLACK_LEVEL from lensless.hardware.sensor import SensorOptions, sensor_dict, SensorParam from fractions import Fraction import time @@ -56,13 +55,14 @@ @hydra.main(version_base=None, config_path="../../configs", config_name="capture") def capture(config): - sensor = config.sensor + black_level, ccm, supported_bit_depth, nbits_measured = check_capture_config(config) + bayer = config.bayer nbits_bayer = config.nbits_bayer fn = config.fn exp = config.exp config_pause = config.config_pause - sensor_mode = config.sensor_mode + sensor_mode = int(config.sensor_mode) rgb = config.rgb gray = config.gray iso = config.iso @@ -71,44 +71,9 @@ def capture(config): res = config.res nbits_out = config.nbits_out - assert sensor in SensorOptions.values(), f"Sensor must be one of {SensorOptions.values()}" - - assert res is None or down is None, "Cannot specify both res and down" - assert rgb is False or gray is False, "Cannot set both rgb and gray" - - supported_bit_depth = sensor_dict[sensor][SensorParam.BIT_DEPTH] - assert ( - nbits_out in supported_bit_depth - ), f"nbits_out must be one of {supported_bit_depth} for sensor {sensor}" - assert nbits_bayer in [8, 16], "nbits_bayer must be 8 or 16" - if nbits_bayer == 16: - # TODO may not work if there are multiple bit depths above 8 - nbits_measured = max(supported_bit_depth) - assert nbits_measured > 8 - else: - nbits_measured = 8 - - if SensorParam.BLACK_LEVEL in sensor_dict[sensor]: - black_level = sensor_dict[sensor][SensorParam.BLACK_LEVEL] * (2**nbits_measured - 1) - else: - black_level = 0 - if SensorParam.CCM_MATRIX in sensor_dict[sensor]: - ccm = sensor_dict[sensor][SensorParam.CCM_MATRIX] - else: - ccm = None - - # https://www.raspberrypi.com/documentation/accessories/camera.html#hardware-specification - sensor_param = sensor_dict[sensor] - assert exp <= sensor_param[SensorParam.MAX_EXPOSURE] - assert exp >= sensor_param[SensorParam.MIN_EXPOSURE] - sensor_mode = int(sensor_mode) - distro = get_distro() print("RPi distribution : {}".format(distro)) - if sensor == SensorOptions.RPI_GS.value: - assert not legacy - if "bullseye" in distro and not legacy: # TODO : grayscale and downsample assert not rgb @@ -161,9 +126,7 @@ def capture(config): fn += ".png" max_res = picam2.camera_properties["PixelArraySize"] - if res: - assert len(res) == 2 - else: + if res is None: res = np.array(max_res) if down is not None: res = (np.array(res) / down).astype(int) @@ -184,7 +147,6 @@ def capture(config): "AnalogueGain": 1.0, } if config.awb_gains is not None: - assert len(config.awb_gains) == 2 new_controls["ColourGains"] = tuple(config.awb_gains) picam2.set_controls(new_controls) @@ -281,9 +243,7 @@ def capture(config): from picamerax import PiCamera camera = PiCamera() - if res: - assert len(res) == 2 - else: + if res is None: res = np.array(camera.MAX_RESOLUTION) if down is not None: res = (np.array(res) / down).astype(int) @@ -295,7 +255,6 @@ def capture(config): time.sleep(config.config_pause) if config.awb_gains is not None: - assert len(config.awb_gains) == 2 g = (Fraction(config.awb_gains[0]), Fraction(config.awb_gains[1])) g = tuple(g) camera.awb_mode = "off" From a942086e91531832fe5f479470cb18cc052117c4 Mon Sep 17 00:00:00 2001 From: Eric Bezzam Date: Wed, 29 Nov 2023 17:36:53 +0100 Subject: [PATCH 06/11] More intuitive capture inputs. --- configs/capture.yaml | 2 +- lensless/hardware/utils.py | 18 +++++------------- scripts/measure/on_device_capture.py | 9 +++++---- 3 files changed, 11 insertions(+), 18 deletions(-) diff --git a/configs/capture.yaml b/configs/capture.yaml index 6392ebc2..8ffdd769 100644 --- a/configs/capture.yaml +++ b/configs/capture.yaml @@ -1,7 +1,7 @@ sensor: rpi_hq legacy: True # Legacy image capture software (raspistill) or new one (libcamera) bayer: True # whether to capture Bayer data, otherwise directly capture RGB (demosaiced) data -nbits_bayer: 16 # output bit depth for Bayer data (16 or 8), only used if legacy=True +nbits_capture: 12 # (if legacy=True) output bit depth for Bayer data. should correspond to supported bit depths: lensless/hardware/sensor.py fn: test exp: 0.5 config_pause: 2 diff --git a/lensless/hardware/utils.py b/lensless/hardware/utils.py index c49cce21..bf97cdbb 100644 --- a/lensless/hardware/utils.py +++ b/lensless/hardware/utils.py @@ -20,14 +20,13 @@ def check_capture_config(config): sensor = config.sensor - nbits_bayer = config.nbits_bayer + nbits_capture = config.nbits_capture exp = config.exp rgb = config.rgb gray = config.gray legacy = config.legacy down = config.down res = config.res - nbits_out = config.nbits_out awb_gains = config.awb_gains assert sensor in SensorOptions.values(), f"Sensor must be one of {SensorOptions.values()}" @@ -40,18 +39,11 @@ def check_capture_config(config): supported_bit_depth = sensor_dict[sensor][SensorParam.BIT_DEPTH] assert ( - nbits_out in supported_bit_depth - ), f"nbits_out must be one of {supported_bit_depth} for sensor {sensor}" - assert nbits_bayer in [8, 16], "nbits_bayer must be 8 or 16" - if nbits_bayer == 16: - # TODO may not work if there are multiple bit depths above 8 - nbits_measured = max(supported_bit_depth) - assert nbits_measured > 8 - else: - nbits_measured = 8 + nbits_capture in supported_bit_depth + ), f"nbits_capture must be one of {supported_bit_depth} for sensor {sensor}" if SensorParam.BLACK_LEVEL in sensor_dict[sensor]: - black_level = sensor_dict[sensor][SensorParam.BLACK_LEVEL] * (2**nbits_measured - 1) + black_level = sensor_dict[sensor][SensorParam.BLACK_LEVEL] * (2**nbits_capture - 1) else: black_level = 0 if SensorParam.CCM_MATRIX in sensor_dict[sensor]: @@ -71,7 +63,7 @@ def check_capture_config(config): assert not legacy, "Legacy capture software not supported for Global Shutter sensor" # return sensor parameters - return black_level, ccm, supported_bit_depth, nbits_measured + return black_level, ccm, supported_bit_depth def capture( diff --git a/scripts/measure/on_device_capture.py b/scripts/measure/on_device_capture.py index 8196a79f..d03e6e3f 100644 --- a/scripts/measure/on_device_capture.py +++ b/scripts/measure/on_device_capture.py @@ -55,10 +55,10 @@ @hydra.main(version_base=None, config_path="../../configs", config_name="capture") def capture(config): - black_level, ccm, supported_bit_depth, nbits_measured = check_capture_config(config) + black_level, ccm, supported_bit_depth = check_capture_config(config) bayer = config.bayer - nbits_bayer = config.nbits_bayer + nbits_capture = config.nbits_capture fn = config.fn exp = config.exp config_pause = config.config_pause @@ -197,9 +197,10 @@ def capture(config): # get bayer data raw_data = np.sum(stream.array, axis=2) - if nbits_bayer == 16: + if nbits_capture > 8: output = raw_data.astype(np.uint16) else: + # TODO may be wrong if multiple bit depths above 8 are supported output = raw_data / (2 ** max(supported_bit_depth) - 1) * 255 output = output.astype(np.uint8) # output = (np.sum(stream.array, axis=2) >> 2).astype(np.uint8) @@ -213,7 +214,7 @@ def capture(config): output_rgb = bayer2rgb_cc( output, - nbits=nbits_measured, + nbits=nbits_capture, blue_gain=blue_gain, red_gain=red_gain, black_level=black_level, From c9e7ce448ea679e71057721954513fdebc9f06a5 Mon Sep 17 00:00:00 2001 From: Eric Bezzam Date: Wed, 29 Nov 2023 17:52:00 +0100 Subject: [PATCH 07/11] Improve remote capture. --- configs/demo.yaml | 27 +++++++++------- lensless/hardware/utils.py | 5 ++- lensless/utils/io.py | 10 +++++- scripts/measure/on_device_capture.py | 1 - scripts/measure/remote_capture.py | 48 ++++++++-------------------- 5 files changed, 42 insertions(+), 49 deletions(-) diff --git a/configs/demo.yaml b/configs/demo.yaml index 47f13ba4..698e73be 100644 --- a/configs/demo.yaml +++ b/configs/demo.yaml @@ -30,23 +30,26 @@ display: white: False capture: - sensor: rpi_hq gamma: null # for visualization - exp: 0.02 - delay: 2 script: ~/LenslessPiCam/scripts/measure/on_device_capture.py - iso: 100 + delay: 2 + + # -- capture settings + sensor: rpi_hq + legacy: True # Legacy image capture software (raspistill) or new one (libcamera) + bayer: True # whether to capture Bayer data, otherwise directly capture RGB (demosaiced) data + nbits_capture: 12 # bit-depth to capture, check support bit depth of sensor: lensless/hardware/sensor.py + raw_data_fn: raw_data # output file name + exp: 0.02 config_pause: 2 - sensor_mode: "0" - # nbits_out: 12 - nbits_out: 8 # light data transer, doesn't seem to worsen performance - nbits: 12 - legacy: True + sensor_mode: "0" # {'off': 0, 'auto': 1, 'sunlight': 2, 'cloudy': 3, 'shade': 4, 'tungsten': 5, 'fluorescent': 6, 'incandescent': 7, 'flash': 8, 'horizon': 9} + iso: 100 + + # -- if bayer=True, capture Bayer data but return as RGB or grayscale gray: False # only for legacy=True, if bayer=True, remote script returns grayscale data # rgb: False # only for legacy=True, if bayer=True, remote script return RGB data - raw_data_fn: raw_data - bayer: True - source: white + # nbits_out: 12 + nbits_out: 8 # used if gray=True or rgb=True # remote script return Bayer data, full res # awb_gains: null diff --git a/lensless/hardware/utils.py b/lensless/hardware/utils.py index bf97cdbb..3ee77b54 100644 --- a/lensless/hardware/utils.py +++ b/lensless/hardware/utils.py @@ -21,12 +21,13 @@ def check_capture_config(config): sensor = config.sensor nbits_capture = config.nbits_capture + nbits_out = config.nbits_out exp = config.exp rgb = config.rgb gray = config.gray legacy = config.legacy down = config.down - res = config.res + res = config.res if "res" in config.keys() else None awb_gains = config.awb_gains assert sensor in SensorOptions.values(), f"Sensor must be one of {SensorOptions.values()}" @@ -42,6 +43,8 @@ def check_capture_config(config): nbits_capture in supported_bit_depth ), f"nbits_capture must be one of {supported_bit_depth} for sensor {sensor}" + assert nbits_capture >= nbits_out, "nbits_capture must be greater than or equal to nbits_out" + if SensorParam.BLACK_LEVEL in sensor_dict[sensor]: black_level = sensor_dict[sensor][SensorParam.BLACK_LEVEL] * (2**nbits_capture - 1) else: diff --git a/lensless/utils/io.py b/lensless/utils/io.py index 750b0e0e..e36c5686 100644 --- a/lensless/utils/io.py +++ b/lensless/utils/io.py @@ -39,6 +39,10 @@ def load_image( """ Load image as numpy array. + Note that for bayer data input, the image is converted to RGB using + color correction matrix (CCM) and black level subtraction parameters + for the Raspberry Pi HQ camera (12-bit). + Parameters ---------- fp : str @@ -122,12 +126,16 @@ def load_image( if bayer: assert len(img.shape) == 2, img.shape + + # assume RPi HQ camera if nbits is None: if img.max() > 255: - # HQ camera nbits = 12 else: nbits = 8 + # convert black level from 12 bit to 8 bit + if black_level > 255: + black_level = black_level / (2**12 - 1) * 255 if back: back_img = cv2.imread(back, cv2.IMREAD_UNCHANGED) diff --git a/scripts/measure/on_device_capture.py b/scripts/measure/on_device_capture.py index d03e6e3f..b6641e3c 100644 --- a/scripts/measure/on_device_capture.py +++ b/scripts/measure/on_device_capture.py @@ -33,7 +33,6 @@ from PIL import Image from lensless.hardware.utils import get_distro, check_capture_config from lensless.utils.image import bayer2rgb_cc, rgb2gray, resize -from lensless.hardware.sensor import SensorOptions, sensor_dict, SensorParam from fractions import Fraction import time diff --git a/scripts/measure/remote_capture.py b/scripts/measure/remote_capture.py index 6777347b..be0d3d68 100644 --- a/scripts/measure/remote_capture.py +++ b/scripts/measure/remote_capture.py @@ -33,8 +33,7 @@ from pprint import pprint import matplotlib.pyplot as plt import rawpy -from lensless.hardware.utils import check_username_hostname -from lensless.hardware.sensor import SensorOptions, sensor_dict, SensorParam +from lensless.hardware.utils import check_username_hostname, check_capture_config from lensless.utils.image import rgb2gray, print_image_info from lensless.utils.plot import plot_image, pixel_histogram from lensless.utils.io import save_image @@ -44,30 +43,21 @@ @hydra.main(version_base=None, config_path="../../configs", config_name="demo") def liveview(config): - sensor = config.capture.sensor - assert sensor in SensorOptions.values(), f"Sensor must be one of {SensorOptions.values()}" + username = config.rpi.username + hostname = config.rpi.hostname + check_username_hostname(username, hostname) + black_level, ccm, _ = check_capture_config(config.capture) + sensor = config.capture.sensor bayer = config.capture.bayer rgb = config.capture.rgb gray = config.capture.gray - - check_username_hostname(config.rpi.username, config.rpi.hostname) - username = config.rpi.username - hostname = config.rpi.hostname legacy = config.capture.legacy nbits_out = config.capture.nbits_out fn = config.capture.raw_data_fn gamma = config.capture.gamma - source = config.capture.source plot = config.plot - assert ( - nbits_out in sensor_dict[sensor][SensorParam.BIT_DEPTH] - ), f"capture.nbits_out must be one of {sensor_dict[sensor][SensorParam.BIT_DEPTH]} for sensor {sensor}" - assert ( - config.capture.nbits in sensor_dict[sensor][SensorParam.BIT_DEPTH] - ), f"capture.nbits must be one of {sensor_dict[sensor][SensorParam.BIT_DEPTH]} for sensor {sensor}" - if config.save: if config.output is not None: # make sure output directory exists @@ -84,10 +74,8 @@ def liveview(config): pic_command = ( f"{config.rpi.python} {config.capture.script} sensor={sensor} bayer={bayer} fn={remote_fn} exp={config.capture.exp} iso={config.capture.iso} " f"config_pause={config.capture.config_pause} sensor_mode={config.capture.sensor_mode} nbits_out={config.capture.nbits_out} " - f"legacy={config.capture.legacy} rgb={config.capture.rgb} gray={config.capture.gray} " + f"legacy={config.capture.legacy} rgb={config.capture.rgb} gray={config.capture.gray} nbits_capture={config.capture.nbits_capture} " ) - if config.capture.nbits > 8: - pic_command += " sixteen=True" if config.capture.down: pic_command += f" down={config.capture.down}" if config.capture.awb_gains: @@ -232,6 +220,8 @@ def liveview(config): blue_gain=blue_gain, red_gain=red_gain, nbits_out=nbits_out, + black_level=black_level, + ccm=ccm, ) # write RGB data @@ -250,22 +240,12 @@ def liveview(config): if save: plt.savefig(os.path.join(save, f"{fn}_plot.png")) - # plot red channel - if source == "red": - img_1chan = img[:, :, 0] - elif source == "green": - img_1chan = img[:, :, 1] - elif source == "blue": - img_1chan = img[:, :, 2] - else: - img_1chan = rgb2gray(img[None, :, :, :]) + # plot grayscale + img_1chan = rgb2gray(img[None, :, :, :]) ax = plot_image(img_1chan) - if source == "white": - ax.set_title("Gray scale") - else: - ax.set_title(f"{source} channel") + ax.set_title("Grayscale") if save: - plt.savefig(os.path.join(save, f"{fn}_1chan.png")) + plt.savefig(os.path.join(save, f"{fn}_gray.png")) # plot histogram, useful for checking clipping pixel_histogram(img) @@ -273,7 +253,7 @@ def liveview(config): plt.savefig(os.path.join(save, f"{fn}_hist.png")) pixel_histogram(img_1chan) if save: - plt.savefig(os.path.join(save, f"{fn}_1chan_hist.png")) + plt.savefig(os.path.join(save, f"{fn}_gray_hist.png")) else: ax = plot_image(img, gamma=gamma) From dd0609a359bdb91272e1b466e52be389764d1a83 Mon Sep 17 00:00:00 2001 From: Eric Bezzam Date: Thu, 30 Nov 2023 14:26:35 +0100 Subject: [PATCH 08/11] Improve remote capture API. --- configs/collect_dataset.yaml | 3 + configs/demo.yaml | 2 +- lensless/hardware/constants.py | 19 --- lensless/hardware/utils.py | 109 ++++++++++++------ lensless/utils/image.py | 2 + lensless/utils/io.py | 12 +- scripts/measure/remote_capture.py | 185 ++---------------------------- 7 files changed, 96 insertions(+), 236 deletions(-) delete mode 100644 lensless/hardware/constants.py diff --git a/configs/collect_dataset.yaml b/configs/collect_dataset.yaml index 1696e153..69c94f81 100644 --- a/configs/collect_dataset.yaml +++ b/configs/collect_dataset.yaml @@ -35,6 +35,9 @@ display: rot90: 3 capture: + sensor: rpi_hq + nbits_capture: 12 + nbits_out: 8 delay: 2 # to allow picture to display config_pause: 2 iso: 100 diff --git a/configs/demo.yaml b/configs/demo.yaml index 698e73be..7fde92e0 100644 --- a/configs/demo.yaml +++ b/configs/demo.yaml @@ -39,7 +39,7 @@ capture: legacy: True # Legacy image capture software (raspistill) or new one (libcamera) bayer: True # whether to capture Bayer data, otherwise directly capture RGB (demosaiced) data nbits_capture: 12 # bit-depth to capture, check support bit depth of sensor: lensless/hardware/sensor.py - raw_data_fn: raw_data # output file name + fn: raw_data # output file name exp: 0.02 config_pause: 2 sensor_mode: "0" # {'off': 0, 'auto': 1, 'sunlight': 2, 'cloudy': 3, 'shade': 4, 'tungsten': 5, 'fluorescent': 6, 'incandescent': 7, 'flash': 8, 'horizon': 9} diff --git a/lensless/hardware/constants.py b/lensless/hardware/constants.py deleted file mode 100644 index 10293949..00000000 --- a/lensless/hardware/constants.py +++ /dev/null @@ -1,19 +0,0 @@ -# ############################################################################# -# constants.py -# ================= -# Authors : -# Eric BEZZAM [ebezzam@gmail.com] -# ############################################################################# - - -import numpy as np - -# estimated here: https://www.strollswithmydog.com/open-raspberry-pi-high-quality-camera-raw -RPI_HQ_CAMERA_CCM_MATRIX = np.array( - [ - [2.0659, -0.93119, -0.13421], - [-0.11615, 1.5593, -0.44314], - [0.073694, -0.4368, 1.3636], - ] -) -RPI_HQ_CAMERA_BLACK_LEVEL = 256.3 diff --git a/lensless/hardware/utils.py b/lensless/hardware/utils.py index 3ee77b54..572d3710 100644 --- a/lensless/hardware/utils.py +++ b/lensless/hardware/utils.py @@ -17,24 +17,39 @@ logging.getLogger("paramiko").setLevel(logging.WARNING) -def check_capture_config(config): - - sensor = config.sensor - nbits_capture = config.nbits_capture - nbits_out = config.nbits_out - exp = config.exp - rgb = config.rgb - gray = config.gray - legacy = config.legacy - down = config.down - res = config.res if "res" in config.keys() else None - awb_gains = config.awb_gains +def check_capture_config( + sensor, + nbits_capture, + nbits_out, + exp, + rgb, + gray, + legacy, + down, + awb_gains, + res=None, + **kwargs, +): + + # sensor = config.sensor + # nbits_capture = config.nbits_capture + # nbits_out = config.nbits_out + # exp = config.exp + # rgb = config.rgb + # gray = config.gray + # legacy = config.legacy + # down = config.down + # res = config.res if "res" in config.keys() else None + # awb_gains = config.awb_gains assert sensor in SensorOptions.values(), f"Sensor must be one of {SensorOptions.values()}" assert res is None or down is None, "Cannot specify both res and down" if res is not None: assert len(res) == 2, "res must be a tuple of length 2" + # if down is not None or res is not None: + # if rgb is False and gray is False: + # warnings.warn("Bayer data will be returned in original dimension, i.e. 'down' and 'res' will be ignored.") assert rgb is False or gray is False, "Cannot set both rgb and gray" @@ -80,16 +95,16 @@ def capture( config_pause=2, sensor_mode="0", nbits_out=12, + nbits_capture=12, legacy=True, rgb=False, gray=False, - nbits=12, down=None, awb_gains=None, rpi_python="~/LenslessPiCam/lensless_env/bin/python", capture_script="~/LenslessPiCam/scripts/measure/on_device_capture.py", verbose=False, - output_path=None, + output_dir=None, **kwargs, ): """ @@ -123,10 +138,10 @@ def capture( Whether to capture RGB image. gray : bool Whether to capture grayscale image. - nbits : int - Number of bits of image. + nbits_capture : int + Number of bits of image captured by sensor. down : int - Downsample factor. + Downsample factor. Only used if non-Bayer data is returned. awb_gains : list AWB gains (red, blue). rpi_python : str @@ -140,6 +155,19 @@ def capture( """ + black_level, ccm, supported_bit_depth = check_capture_config( + sensor=sensor, + nbits_capture=nbits_capture, + nbits_out=nbits_out, + exp=exp, + rgb=rgb, + gray=gray, + legacy=legacy, + down=down, + awb_gains=awb_gains, + **kwargs, + ) + # check_username_hostname(rpi_username, rpi_hostname) assert sensor in SensorOptions.values(), f"Sensor must be one of {SensorOptions.values()}" @@ -148,10 +176,8 @@ def capture( pic_command = ( f"{rpi_python} {capture_script} sensor={sensor} bayer={bayer} fn={remote_fn} exp={exp} iso={iso} " f"config_pause={config_pause} sensor_mode={sensor_mode} nbits_out={nbits_out} " - f"legacy={legacy} rgb={rgb} gray={gray} " + f"legacy={legacy} rgb={rgb} gray={gray} nbits_capture={nbits_capture} " ) - if nbits > 8: - pic_command += " sixteen=True" if down: pic_command += f" down={down}" if awb_gains: @@ -203,8 +229,8 @@ def capture( # copy over DNG file remotefile = f"~/{remote_fn}.dng" localfile = f"{fn}.dng" - if output_path is not None: - localfile = os.path.join(output_path, localfile) + if output_dir is not None: + localfile = os.path.join(output_dir, localfile) if verbose: print(f"\nCopying over picture as {localfile}...") os.system( @@ -212,7 +238,15 @@ def capture( % (rpi_username, rpi_hostname, remotefile, localfile) ) - img = load_image(localfile, verbose=True, bayer=bayer, nbits_out=nbits_out) + img = load_image( + localfile, + verbose=True, + bayer=bayer, + nbits_out=nbits_out, + black_level=black_level, + ccm=ccm, + downsample=down, + ) # print image properties print_image_info(img) @@ -226,8 +260,8 @@ def capture( remotefile = f"~/{remote_fn}.png" localfile = f"{fn}.png" - if output_path is not None: - localfile = os.path.join(output_path, localfile) + if output_dir is not None: + localfile = os.path.join(output_dir, localfile) if verbose: print(f"\nCopying over picture as {localfile}...") os.system( @@ -243,8 +277,8 @@ def capture( # more pythonic? https://stackoverflow.com/questions/250283/how-to-scp-in-python remotefile = f"~/{remote_fn}.png" localfile = f"{fn}.png" - if output_path is not None: - localfile = os.path.join(output_path, localfile) + if output_dir is not None: + localfile = os.path.join(output_dir, localfile) if verbose: print(f"\nCopying over picture as {localfile}...") os.system( @@ -257,14 +291,16 @@ def capture( else: - if not bayer: - # red_gain = config.camera.red_gain - # blue_gain = config.camera.blue_gain - red_gain = awb_gains[0] - blue_gain = awb_gains[1] - else: - red_gain = None - blue_gain = None + red_gain = awb_gains[0] + blue_gain = awb_gains[1] + # if not bayer: + # # red_gain = config.camera.red_gain + # # blue_gain = config.camera.blue_gain + # red_gain = awb_gains[0] + # blue_gain = awb_gains[1] + # else: + # red_gain = None + # blue_gain = None # load image if verbose: @@ -277,6 +313,9 @@ def capture( blue_gain=blue_gain, red_gain=red_gain, nbits_out=nbits_out, + black_level=black_level, + ccm=ccm, + downsample=down, ) # write RGB data diff --git a/lensless/utils/image.py b/lensless/utils/image.py index e9be1a57..726e193e 100644 --- a/lensless/utils/image.py +++ b/lensless/utils/image.py @@ -270,6 +270,8 @@ def bayer2rgb_cc( if ccm is None: ccm = np.eye(3) + else: + assert ccm.shape == (3, 3) ccm = ccm.copy().astype(np.float32) # demosaic Bayer data diff --git a/lensless/utils/io.py b/lensless/utils/io.py index e36c5686..947ad1a3 100644 --- a/lensless/utils/io.py +++ b/lensless/utils/io.py @@ -13,7 +13,6 @@ import os.path from lensless.utils.plot import plot_image -from lensless.hardware.constants import RPI_HQ_CAMERA_BLACK_LEVEL, RPI_HQ_CAMERA_CCM_MATRIX from lensless.utils.image import bayer2rgb_cc, print_image_info, resize, rgb2gray, get_max_val @@ -22,10 +21,10 @@ def load_image( verbose=False, flip=False, bayer=False, - black_level=RPI_HQ_CAMERA_BLACK_LEVEL, + black_level=0, blue_gain=None, red_gain=None, - ccm=RPI_HQ_CAMERA_CCM_MATRIX, + ccm=None, back=None, nbits_out=None, as_4d=False, @@ -58,9 +57,9 @@ def load_image( red_gain : float Red gain for color correction. black_level : float - Black level. Default is to use that of Raspberry Pi HQ camera. + Black level to remove. ccm : :py:class:`~numpy.ndarray` - Color correction matrix. Default is to use that of Raspberry Pi HQ camera. + Color correction matrix. back : array_like Background level to subtract. nbits_out : int @@ -133,9 +132,6 @@ def load_image( nbits = 12 else: nbits = 8 - # convert black level from 12 bit to 8 bit - if black_level > 255: - black_level = black_level / (2**12 - 1) * 255 if back: back_img = cv2.imread(back, cv2.IMREAD_UNCHANGED) diff --git a/scripts/measure/remote_capture.py b/scripts/measure/remote_capture.py index be0d3d68..93e4ae67 100644 --- a/scripts/measure/remote_capture.py +++ b/scripts/measure/remote_capture.py @@ -28,16 +28,11 @@ import hydra import os -import subprocess -import cv2 -from pprint import pprint import matplotlib.pyplot as plt -import rawpy -from lensless.hardware.utils import check_username_hostname, check_capture_config -from lensless.utils.image import rgb2gray, print_image_info +from lensless.hardware.utils import check_username_hostname, capture +from lensless.utils.image import rgb2gray from lensless.utils.plot import plot_image, pixel_histogram from lensless.utils.io import save_image -from lensless.utils.io import load_image @hydra.main(version_base=None, config_path="../../configs", config_name="demo") @@ -47,14 +42,9 @@ def liveview(config): hostname = config.rpi.hostname check_username_hostname(username, hostname) - black_level, ccm, _ = check_capture_config(config.capture) - sensor = config.capture.sensor - bayer = config.capture.bayer - rgb = config.capture.rgb + # black_level, ccm, _ = check_capture_config(config.capture) gray = config.capture.gray - legacy = config.capture.legacy - nbits_out = config.capture.nbits_out - fn = config.capture.raw_data_fn + fn = config.capture.fn gamma = config.capture.gamma plot = config.plot @@ -69,167 +59,16 @@ def liveview(config): save = False # take picture - remote_fn = "remote_capture" - print("\nTaking picture...") - pic_command = ( - f"{config.rpi.python} {config.capture.script} sensor={sensor} bayer={bayer} fn={remote_fn} exp={config.capture.exp} iso={config.capture.iso} " - f"config_pause={config.capture.config_pause} sensor_mode={config.capture.sensor_mode} nbits_out={config.capture.nbits_out} " - f"legacy={config.capture.legacy} rgb={config.capture.rgb} gray={config.capture.gray} nbits_capture={config.capture.nbits_capture} " + _, img = capture( + rpi_username=username, + rpi_hostname=hostname, + verbose=True, + output_dir=save, + **config.capture, ) - if config.capture.down: - pic_command += f" down={config.capture.down}" - if config.capture.awb_gains: - pic_command += f" awb_gains=[{config.capture.awb_gains[0]},{config.capture.awb_gains[1]}]" - - print(f"COMMAND : {pic_command}") - ssh = subprocess.Popen( - ["ssh", "%s@%s" % (username, hostname), pic_command], - shell=False, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - ) - result = ssh.stdout.readlines() - error = ssh.stderr.readlines() - - if error != [] and legacy: # new camera software seems to return error even if it works - print("ERROR: %s" % error) - return - if result == []: - error = ssh.stderr.readlines() - print("ERROR: %s" % error) - return - else: - result = [res.decode("UTF-8") for res in result] - result = [res for res in result if len(res) > 3] - result_dict = dict() - for res in result: - _key = res.split(":")[0].strip() - _val = "".join(res.split(":")[1:]).strip() - result_dict[_key] = _val - # result_dict = dict(map(lambda s: map(str.strip, s.split(":")), result)) - print("COMMAND OUTPUT : ") - pprint(result_dict) - - if ( - "RPi distribution" in result_dict.keys() - and "bullseye" in result_dict["RPi distribution"] - and not legacy - ): - - if bayer: - - # copy over DNG file - remotefile = f"~/{remote_fn}.dng" - localfile = os.path.join(save, f"{fn}.dng") - print(f"\nCopying over picture as {localfile}...") - os.system('scp "%s@%s:%s" %s' % (username, hostname, remotefile, localfile)) - - img = load_image(localfile, verbose=True, bayer=bayer, nbits_out=nbits_out) - - # raw = rawpy.imread(localfile) - - # # https://letmaik.github.io/rawpy/api/rawpy.Params.html#rawpy.Params - # # https://www.libraw.org/docs/API-datastruct-eng.html - # if nbits_out > 8: - # # only 8 or 16 bit supported by postprocess - # if nbits_out != 16: - # print("casting to 16 bit...") - # output_bps = 16 - # else: - # if nbits_out != 8: - # print("casting to 8 bit...") - # output_bps = 8 - # img = raw.postprocess( - # adjust_maximum_thr=0, # default 0.75 - # no_auto_scale=False, - # # no_auto_scale=True, - # gamma=(1, 1), - # output_bps=output_bps, - # bright=1, # default 1 - # exp_shift=1, - # no_auto_bright=True, - # # use_camera_wb=True, - # # use_auto_wb=False, - # # -- gives better balance for PSF measurement - # use_camera_wb=False, - # use_auto_wb=True, # default is False? f both use_camera_wb and use_auto_wb are True, then use_auto_wb has priority. - # ) - - # print image properties - print_image_info(img) - - # save as PNG - png_out = f"{fn}.png" - print(f"Saving RGB file as: {png_out}") - cv2.imwrite(png_out, cv2.cvtColor(img, cv2.COLOR_RGB2BGR)) - - else: - - remotefile = f"~/{remote_fn}.png" - localfile = f"{fn}.png" - if save: - localfile = os.path.join(save, localfile) - print(f"\nCopying over picture as {localfile}...") - os.system('scp "%s@%s:%s" %s' % (username, hostname, remotefile, localfile)) - - img = load_image(localfile, verbose=True) - - # legacy software running on RPi - else: - # copy over file - # more pythonic? https://stackoverflow.com/questions/250283/how-to-scp-in-python - remotefile = f"~/{remote_fn}.png" - localfile = f"{fn}.png" - if save: - localfile = os.path.join(save, localfile) - print(f"\nCopying over picture as {localfile}...") - os.system('scp "%s@%s:%s" %s' % (username, hostname, remotefile, localfile)) - - if rgb or gray: - img = load_image(localfile, verbose=True) - - else: - - if not bayer: - red_gain = config.camera.red_gain - blue_gain = config.camera.blue_gain - else: - red_gain = None - blue_gain = None - # # get white balance gains - # if red_gain is None: - # red_gain = float(result_dict["Red gain"]) - # if blue_gain is None: - # blue_gain = float(result_dict["Blue gain"]) - - # # get white balance gains - # if bayer: - # red_gain = 1 - # blue_gain = 1 - # else: - # red_gain = float(result_dict["Red gain"]) - # blue_gain = float(result_dict["Blue gain"]) - - # load image - print("\nLoading picture...") - - img = load_image( - localfile, - verbose=True, - bayer=bayer, - blue_gain=blue_gain, - red_gain=red_gain, - nbits_out=nbits_out, - black_level=black_level, - ccm=ccm, - ) - - # write RGB data - if not bayer: - cv2.imwrite(localfile, cv2.cvtColor(img, cv2.COLOR_RGB2BGR)) # save image as viewable 8 bit - fp = os.path.join(save, f"{fn}_8bit.png") + fp = os.path.join(save, f"{fn}_rgb_8bit.png") save_image(img, fp) # plot RGB @@ -266,7 +105,7 @@ def liveview(config): plt.show() if save: - print(f"\nSaved plots to: {save}") + print(f"\nSaved images to: {save}") if __name__ == "__main__": From 4503a65e005f307a70ddb884f648ee8af56b835e Mon Sep 17 00:00:00 2001 From: Eric Bezzam Date: Thu, 30 Nov 2023 14:27:11 +0100 Subject: [PATCH 09/11] Improve capture script. --- scripts/measure/collect_dataset_on_device.py | 29 +++++++++++++------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/scripts/measure/collect_dataset_on_device.py b/scripts/measure/collect_dataset_on_device.py index 601a503b..1ef4597d 100644 --- a/scripts/measure/collect_dataset_on_device.py +++ b/scripts/measure/collect_dataset_on_device.py @@ -21,13 +21,8 @@ import tqdm from picamerax import PiCamera from fractions import Fraction -from PIL import Image from lensless.utils.io import save_image - -from lensless.hardware.constants import ( - RPI_HQ_CAMERA_CCM_MATRIX, - RPI_HQ_CAMERA_BLACK_LEVEL, -) +from lensless.hardware.sensor import SensorOptions, sensor_dict, SensorParam import picamerax.array from lensless.utils.image import bayer2rgb_cc, resize import cv2 @@ -46,6 +41,20 @@ def collect_dataset(config): MIN_LEVEL = config.min_level MAX_LEVEL = config.max_level + # get sensor parameters + sensor = config.capture.sensor + nbits_capture = config.capture.nbits_capture + assert sensor in SensorOptions.values(), f"Sensor must be one of {SensorOptions.values()}" + if SensorParam.BLACK_LEVEL in sensor_dict[sensor]: + black_level = sensor_dict[sensor][SensorParam.BLACK_LEVEL] * (2**nbits_capture - 1) + else: + black_level = 0 + if SensorParam.CCM_MATRIX in sensor_dict[sensor]: + ccm = sensor_dict[sensor][SensorParam.CCM_MATRIX] + else: + ccm = None + assert sensor == "rpi_hq", "Only RPi HQ camera supported for now!" + # if output dir exists check how many files done print(f"Output directory : {output_dir}") start_idx = 0 @@ -205,12 +214,12 @@ def collect_dataset(config): # convert to RGB output = bayer2rgb_cc( output_bayer, - nbits=12, + nbits=nbits_capture, blue_gain=float(g[1]), red_gain=float(g[0]), - black_level=RPI_HQ_CAMERA_BLACK_LEVEL, - ccm=RPI_HQ_CAMERA_CCM_MATRIX, - nbits_out=8, + black_level=black_level, + ccm=ccm, + nbits_out=config.capture.nbits_out, ) if down: From ffe9b36e31a3514fddde3e20989c029a8977786b Mon Sep 17 00:00:00 2001 From: Eric Bezzam Date: Thu, 30 Nov 2023 14:31:27 +0100 Subject: [PATCH 10/11] Update config check. --- scripts/measure/on_device_capture.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/measure/on_device_capture.py b/scripts/measure/on_device_capture.py index b6641e3c..93d32b1c 100644 --- a/scripts/measure/on_device_capture.py +++ b/scripts/measure/on_device_capture.py @@ -54,7 +54,7 @@ @hydra.main(version_base=None, config_path="../../configs", config_name="capture") def capture(config): - black_level, ccm, supported_bit_depth = check_capture_config(config) + black_level, ccm, supported_bit_depth = check_capture_config(**config) bayer = config.bayer nbits_capture = config.nbits_capture From 78f083c73069c8e2c020e4206ecfb25a813b97e9 Mon Sep 17 00:00:00 2001 From: Eric Bezzam Date: Thu, 30 Nov 2023 14:35:23 +0100 Subject: [PATCH 11/11] Better commentsfor capture config. --- configs/capture.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/configs/capture.yaml b/configs/capture.yaml index 8ffdd769..836a1ac8 100644 --- a/configs/capture.yaml +++ b/configs/capture.yaml @@ -7,14 +7,14 @@ exp: 0.5 config_pause: 2 sensor_mode: "0" # {'off': 0, 'auto': 1, 'sunlight': 2, 'cloudy': 3, 'shade': 4, 'tungsten': 5, 'fluorescent': 6, 'incandescent': 7, 'flash': 8, 'horizon': 9} iso: 100 -# awb_gains: null # to use gains estimte by system +# awb_gains: null # to use gains estimated by system awb_gains: [1.9, 1.2] # -- only one of below should be set down: null # downsample from maximum resolution res: null # desired resolution -# -- below only used if bayer=True, capture Bayer data but return as RGB or grayscale +# -- below only used if bayer=True, i.e. capture Bayer data but return as RGB/grayscale rgb: False gray: False nbits_out: 8 \ No newline at end of file