diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml new file mode 100644 index 0000000..dea072a --- /dev/null +++ b/.github/workflows/docs.yaml @@ -0,0 +1,47 @@ +name: docs + +on: + push: + branches: [ master ] + +jobs: + build-and-deploy: + name: build and deploy docs + runs-on: ubuntu-latest + env: + MPLBACKEND: 'Agg' + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python 3.11 + uses: actions/setup-python@v4 + with: + python-version: 3.11 + + - name: Install OQ and dependencies + run: | + sudo apt-get install -y libspatialindex-dev + git clone --depth=1 https://github.com/gem/oq-engine.git + cd oq-engine + python ./install.py devel + cd .. + source $HOME/openquake/bin/activate + pip install h3 + pip install -r requirements.txt + pip install -e . + pip install sphinx furo recommonmark + + - name: Build HTML docs + run: | + source $HOME/openquake/bin/activate + sphinx-build -b html doc_src/source doc_src/build/html + touch doc_src/build/html/.nojekyll + + - name: Deploy to GitHub Pages + uses: peaceiris/actions-gh-pages@v4 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + publish_branch: gh-pages + publish_dir: doc_src/build/html/ + commit_message: "deploy docs from master" diff --git a/.github/workflows/docs_preview.yaml b/.github/workflows/docs_preview.yaml new file mode 100644 index 0000000..68d7215 --- /dev/null +++ b/.github/workflows/docs_preview.yaml @@ -0,0 +1,48 @@ +name: docs preview + +on: + push: + branches: [ docs ] + +jobs: + build-and-deploy: + name: build and deploy docs preview + runs-on: ubuntu-latest + env: + MPLBACKEND: 'Agg' + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python 3.11 + uses: actions/setup-python@v4 + with: + python-version: 3.11 + + - name: Install OQ and dependencies + run: | + sudo apt-get install -y libspatialindex-dev + git clone --depth=1 https://github.com/gem/oq-engine.git + cd oq-engine + python ./install.py devel + cd .. + source $HOME/openquake/bin/activate + pip install h3 + pip install -r requirements.txt + pip install -e . + pip install sphinx furo recommonmark + + - name: Build HTML docs + run: | + source $HOME/openquake/bin/activate + sphinx-build -b html doc_src/source doc_src/build/html + touch doc_src/build/html/.nojekyll + + - name: Deploy preview to gh-pages/preview/docs + uses: peaceiris/actions-gh-pages@v4 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + publish_branch: gh-pages + publish_dir: doc_src/build/html/ + destination_dir: preview/docs + commit_message: "docs preview from docs branch" diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml new file mode 100644 index 0000000..d5eb0f4 --- /dev/null +++ b/.github/workflows/test.yaml @@ -0,0 +1,37 @@ +name: tests + +on: + push: + branches: [ master ] + pull_request: + branches: [ master ] + schedule: + - cron: "0 1 * * *" + +jobs: + test: + name: pytest + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + - name: Set up Python 3.11 + uses: actions/setup-python@v4 + with: + python-version: 3.11 + - name: Install OQ and dependencies + run: | + sudo apt-get install libspatialindex-dev + git clone --depth=1 https://github.com/gem/oq-engine.git + cd oq-engine + python ./install.py devel + cd .. + source $HOME/openquake/bin/activate + pip install h3 + pip install -r requirements.txt + pip install -e . + deactivate + - name: Run test with pytest + run: | + source $HOME/openquake/bin/activate + pytest openquake tests -W ignore::UserWarning diff --git a/.github/workflows/test_page.yaml b/.github/workflows/test_page.yaml deleted file mode 100644 index 60352d6..0000000 --- a/.github/workflows/test_page.yaml +++ /dev/null @@ -1,70 +0,0 @@ -name: test and make pages - -on: - push: - branches: [ master ] - pull_request: - branches: [ master ] - schedule: - - cron: "0 1 * * *" - -jobs: - - test: - name: hazard test - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v4 - - name: Set up Python 3.11 - uses: actions/setup-python@v4 - with: - python-version: 3.11 - - name: Install OQ and dependencies - run: | - sudo apt-get install libspatialindex-dev - git clone --depth=1 https://github.com/gem/oq-engine.git - cd oq-engine - python ./install.py devel - cd .. - source $HOME/openquake/bin/activate - pip install h3 - pip install -r requirements.txt - pip install -e . - deactivate - - name: Run test with pytest - run: | - source $HOME/openquake/bin/activate - pytest openquake tests -W ignore::UserWarning - - pages: - name: pages - needs: test - runs-on: ubuntu-latest - env: - MPLBACKEND: 'Agg' - - steps: - - uses: actions/checkout@v4 - - name: Set up Python 3.11 - uses: actions/setup-python@v4 - with: - python-version: 3.11 - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install sphinx - pip install recommonmark - - name: Make html pages - run: | - cd doc_src ; sphinx-apidoc -o source/ ../openquake ; make html - touch build/html/.nojekyll - - - name: Deploy to GitHub Pages - if: success() - uses: crazy-max/ghaction-github-pages@v2 - with: - target_branch: gh-pages - build_dir: doc_src/build/html/ - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.gitignore b/.gitignore index 8bb9013..52694a8 100644 --- a/.gitignore +++ b/.gitignore @@ -70,6 +70,8 @@ instance/ # Sphinx documentation docs/_build/ +doc_src/build/ +doc_src/source/_api/ # PyBuilder target/ diff --git a/README.md b/README.md index fcd2861..e449031 100644 --- a/README.md +++ b/README.md @@ -1,87 +1,67 @@ # Hamlet: Hazard Model Evaluation and Testing -Hamlet (`openquake.hme`) is a Python package developed (OK, in -development) for qualitative and quantitative evaluation of -Probabilistic Seismic Hazard Analysis (PSHA) models, with the intention -of providing feedback to modelers during the model construction process, -to aid model development. Hamlet is developed by the [GEM +Hamlet (`openquake.hme`) is a Python package for qualitative and quantitative +evaluation of Probabilistic Seismic Hazard Analysis (PSHA) models, with the +intention of providing feedback to modelers during the model construction +process, to aid model development. Hamlet is developed by the [GEM Foundation](https://www.globalquakemodel.org), and uses the [OpenQuake](https://github.com/GEM/oq-engine) software extensively. -Hamlet will incorporate several model test frameworks, including those -developed by GEM and some of those developed outside of GEM such as the -[RELM](http://cseptesting.org/documents/relm.php) tests. Currently, and -likely in the future, the model files will be required to be in the -[OpenQuake](https://github.com/GEM/oq-engine) format, regardless of the -format of their original implementation. - -Most of the Hamlet evaluations are spatial in nature; the model domain -is discretized into grid cells, and comparisons between observations and -model predictions are performed in each grid cell, to highlight where in -the domain the model matches the observations, and where it might need -some refinement. - -Additionally, unlike some other hazard model testing frameworks, Hamlet -is designed to operate on separate components of a hazard model, so that -each component can be evaluated against its corresponding data. For -example, each branch of a source model logic tree can be tested -independently, and each type of source (e.g., subduction megathrust, -crustal, in-slab) can be tested independently as well, in the spatial -framework described above. +Hamlet incorporates several model test frameworks, including those developed by +GEM and some of those developed outside of GEM such as the +[RELM/CSEP](http://cseptesting.org/documents/relm.php) tests. The model files +must be in the [OpenQuake](https://github.com/GEM/oq-engine) format. + +Most of the Hamlet evaluations are spatial in nature; the model domain is +discretized into grid cells, and comparisons between observations and model +predictions are performed in each grid cell, to highlight where in the domain +the model matches the observations, and where it might need some refinement. + +Additionally, unlike some other hazard model testing frameworks, Hamlet is +designed to operate on separate components of a hazard model, so that each +component can be evaluated against its corresponding data. For example, each +branch of a source model logic tree can be tested independently, and each type +of source (e.g., subduction megathrust, crustal, in-slab) can be tested +independently as well, in the spatial framework described above. ## Quickstart ### Installation -Hamlet requires installation Python v.3.7+, the -[OpenQuake](https://github.com/GEM/oq-engine) engine, and some -additional dependencies as well. These are specified in the -`requirements.txt` file. - -(*Note:* A few of the dependencies might be challenging to install. -These are [Rtree](https://toblerity.org/rtree/) and -[h3-py](https://github.com/uber/h3-py). You may have to install -`libspatialindex` or `libspatialindex-dev` on Linux or MacOS first, -depending on your system, for `Rtree`. `h3-py` requires `cc` and `make`, -but then on Linux/MacOS can be installed easily. Please see the -documentation for each.) - -First, install the [OpenQuake](https://github.com/GEM/oq-engine) engine, -following directions on that website. You probably want to install it -into a virtual environment, and you may even want to have a separate -virtual environment for running Hamlet than the -[OpenQuake](https://github.com/GEM/oq-engine) virtual environment that -you normally use (this is up to you). +Hamlet requires Python 3.11+ and the +[OpenQuake](https://github.com/GEM/oq-engine) engine. + +First, install the OpenQuake engine, following directions on that website. You +probably want to install it into a virtual environment. Then, clone the Hamlet repository, and from that directory, install the requirements: ```bash - pip install -r requirements.txt +pip install -r requirements.txt ``` and then install Hamlet: ```bash - pip install -e . +pip install -e . ``` ### Running Hamlet -Hamlet only requires a seismic hazard model (implemented in -[OpenQuake](https://github.com/GEM/oq-engine)) and a processed seismic -catalog (declustered, and ideally classified by source type) to run. -Once installed, Hamlet can be run from the command line: +Hamlet requires a seismic hazard model (implemented in OpenQuake) and a +processed seismic catalog (declustered, and ideally classified by source type) +to run. Once installed, Hamlet can be run from the command line: ```bash - hamlet test_model.yml +hamlet test_model.yml ``` -`test_model.yml` is a configuration file in -[YAML](https://yaml.org) format that specifies the source model, seismic -catalog, tests to be run, and other variables and parameters. +`test_model.yml` is a configuration file in [YAML](https://yaml.org) format +that specifies the source model, seismic catalog, tests to be run, and other +variables and parameters. ## Documentation -Hamlet documentation can currently be found at -https://cossatot.gitlab.io/hamlet/ . +Hamlet documentation can be found at +https://gemsciencetools.github.io/hamlet/ . diff --git a/doc_src/source/_static/custom.css b/doc_src/source/_static/custom.css new file mode 100644 index 0000000..cc53c81 --- /dev/null +++ b/doc_src/source/_static/custom.css @@ -0,0 +1,10 @@ +/* Expand top two levels of sidebar nav by default */ +.sidebar-tree .toctree-l1 > ul, +.sidebar-tree .toctree-l2 > ul { + display: block !important; +} + +.sidebar-tree .toctree-l1 > label, +.sidebar-tree .toctree-l2 > label { + display: none; +} diff --git a/doc_src/source/api.rst b/doc_src/source/api.rst new file mode 100644 index 0000000..5e59d5f --- /dev/null +++ b/doc_src/source/api.rst @@ -0,0 +1,12 @@ +============= +API Reference +============= + +.. autosummary:: + :toctree: _api + :recursive: + + openquake.hme.core + openquake.hme.model_test_frameworks + openquake.hme.utils + openquake.hme.reporting diff --git a/doc_src/source/architecture.rst b/doc_src/source/architecture.rst index 66ec968..49c7e1f 100644 --- a/doc_src/source/architecture.rst +++ b/doc_src/source/architecture.rst @@ -2,19 +2,16 @@ Hamlet Architecture and Testing Workflow ######################################## -Currently, Hamlet performs spatial-temporal hazard model checks and -statistical evaluation of model consistency and performance against an observed -earthquake catalog. +Hamlet performs spatial-temporal hazard model checks and statistical evaluation +of model consistency and performance against an observed earthquake catalog. -Hamlet follows this work process: +Hamlet follows this workflow: 1. Read in :doc:`YAML configuration file <./yaml_config_file>`, that specifies: - - Which tests to be run + - Which tests and evaluations to run, and their parameters - - What parameters for each test - - - What input files: + - Input files: - Seismic Source Model files @@ -28,38 +25,64 @@ Hamlet follows this work process: - GIS files -2. Reads and process SSM: + - JSON results + +2. Read and process SSM: - 1. Loads sources from a single logic tree branch + 1. Load sources from a single logic tree branch (or iterate over all + branches if ``branch: iterate`` is set) - 2. Sorts sources based on their type, with a list for each + 2. Sort sources based on their type -3. Sorts the ruptures from all sources by magnitude and into spatial bins: +3. Sort the ruptures from all sources by magnitude and into spatial bins: - - Makes :class:`~openquake.hme.utils.bins.SpacemagBin` class that holds - ruptures, observed earthquakes, and both model and empirical - Magnitude-Frequency distributions for each bin. + - Uses `H3 `_ hexagonal cells (by default) + to spatially bin ruptures and observed earthquakes -4. Runs the tests: + - Computes both model and empirical Magnitude-Frequency Distributions (MFDs) + for each spatial bin + +4. Run the tests and evaluations: - Basic sanity checks (e.g., whether the observed earthquake maximum magnitude exceeds the model maximum magnitude in each spatial bin) - - Statistical evaluation (i.e., model likelihoods based on the calculated - probabilities of observing the earthquakes in a catalog given the SSM) + - Statistical consistency tests (N-test, M-test, S-test, L-test, + comparing count, magnitude, spatial, and likelihood consistency) + + - Evaluations (MFD comparisons, moment rate analysis, rupture + matching, ground motion evaluation) + + - Multiple tests and evaluations can be run sequentially, and multiple + frameworks can be used in a single run + +5. Write output: + + - HTML reports summarizing the results with maps, plots, and tables + + - GIS files (GeoJSON) with the test results for each spatial bin + + - JSON files with structured results + + +Branch Iteration +================ + +When ``branch: iterate`` is set in the configuration, Hamlet evaluates each +logic tree branch independently in a single run: + +1. The earthquake catalog is loaded once (shared across all branches) - - Multiple tests can be run sequentially, without reloading the SSM. +2. The complete source model is loaded, and all branches are extracted -5. Print/write output. +3. For each branch: - - HTML reports summarizing the results + a. Ruptures are processed for that branch only - - GIS files with the test results for each bin + b. Input data is grouped into spatial bins - - CSV files of the total model and catalog (within the source bins) MFDs + c. All configured tests and evaluations are run + d. Results are collected and memory is freed -At this pre-release stage, most of the development has focused on writing the -test framework, rather than creating a broad suite of tests. However, the -framework is functional at this point, and the development of a test suite is -the next priority. +4. Results from all branches are combined into a single report diff --git a/doc_src/source/conf.py b/doc_src/source/conf.py index 8ed96dd..be1c020 100644 --- a/doc_src/source/conf.py +++ b/doc_src/source/conf.py @@ -18,7 +18,7 @@ # -- Project information ----------------------------------------------------- project = "Hamlet" -copyright = "2019-2021, Richard Styron (GEM Foundation)" +copyright = "2019-2026, Richard Styron (GEM Foundation)" author = "Richard Styron (GEM Foundation)" # The full version, including alpha/beta/rc tags @@ -36,11 +36,14 @@ # ones. extensions = [ "sphinx.ext.autodoc", + "sphinx.ext.autosummary", #'sphinx.ext.intersphinx', "recommonmark", "sphinx.ext.mathjax", ] +autosummary_generate = True + # intersphinx_mapping = {''} # Add any paths that contain templates here, relative to this directory. @@ -49,16 +52,19 @@ # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. # This pattern also affects html_static_path and html_extra_path. -exclude_patterns = [] +exclude_patterns = [ + "model_test_frameworks/gem_tests.rst", # included via .. include:: +] # -- Options for HTML output ------------------------------------------------- # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. # -html_theme = "bizstyle" +html_theme = "furo" # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". html_static_path = ["_static"] +html_css_files = ["custom.css"] diff --git a/doc_src/source/example_configs/minimal_config.yml b/doc_src/source/example_configs/minimal_config.yml new file mode 100644 index 0000000..0f8cb17 --- /dev/null +++ b/doc_src/source/example_configs/minimal_config.yml @@ -0,0 +1,52 @@ +############################################################################### +# Hamlet Minimal Configuration Example +# +# This is the simplest useful configuration: it runs the model MFD comparison, +# the N-test, and the maximum magnitude sanity check for a single logic tree +# branch. +############################################################################### + +meta: + model_name: My Model + description: "Minimal example evaluation" + +config: + model_framework: + gem: + model_mfd: {} + max_mag_check: + append_check: True + warn: True + N_test: + conf_interval: 0.95 + prob_model: poisson + + parallel: False + +input: + bins: + mfd_bin_min: 6.0 + mfd_bin_max: 9.0 + mfd_bin_width: 0.2 + + ssm: + ssm_dir: path/to/ssm/ + ssm_lt_file: ssmLT.xml + branch: b1 + tectonic_region_types: + - Active Shallow Crust + + seis_catalog: + seis_catalog_file: path/to/catalog.csv + stop_date: 2018-01-01 + duration: 40. + columns: + event_id: eventID + time: + - year + - month + - day + +report: + basic: + outfile: outputs/report.html diff --git a/doc_src/source/example_configs/reference_config.yml b/doc_src/source/example_configs/reference_config.yml new file mode 100644 index 0000000..c572118 --- /dev/null +++ b/doc_src/source/example_configs/reference_config.yml @@ -0,0 +1,345 @@ +############################################################################### +# Hamlet Reference Configuration +# +# This file documents every available configuration option. It is not meant +# to be used as-is; copy it and remove or modify the sections you need. +# Options marked [optional] can be omitted; defaults are noted where they exist. +# Many defaults are defined in openquake.hme.core.core.cfg_defaults and are +# applied automatically when not specified. +############################################################################### + +# --------------------------------------------------------------------------- +# Metadata [optional] +# Used in report titles and descriptions. +# --------------------------------------------------------------------------- +meta: + model_name: My Hazard Model + description: "Description of the model being evaluated" + +# --------------------------------------------------------------------------- +# Test and Evaluation Configuration +# --------------------------------------------------------------------------- +config: + + # ------------------------------------------------------------------------- + # model_framework + # + # A dictionary of frameworks to run. Each framework contains its own + # tests and evaluations as keys. Multiple frameworks can be run in a + # single Hamlet run. Available frameworks: gem, relm, sanity. + # ------------------------------------------------------------------------- + model_framework: + + # ----------------------------------------------------------------------- + # GEM Framework + # Tests developed by GEM, some based on the literature, some original. + # ----------------------------------------------------------------------- + gem: + + # Model MFD Evaluation + # Compares the total model magnitude-frequency distribution to the + # observed MFD from the earthquake catalog. Produces a figure in + # the report. Use {} for default parameters. + model_mfd: + investigation_time: 40. # [optional] Duration in years; defaults + # to seis_catalog duration + annualize: True # [optional] Annualize rates; default True + + # Maximum Magnitude Check (sanity check) + # Checks whether the model can produce earthquakes as large as the + # largest observed earthquake in each spatial cell. + max_mag_check: + append_check: True # [optional] Append pass/fail to bin data + warn: True # [optional] Log warnings for failures + + # N-Test: Number of Earthquakes Test + # Compares the total observed earthquake count to the model prediction. + N_test: + conf_interval: 0.95 # Confidence interval for the test + prob_model: poisson # Probability model: "poisson" or "poisson_cum" + prospective: False # [optional] Use prospective catalog; default False + # investigation_time: 40. # [optional] Overridden by catalog duration + # when not prospective + + # M-Test: Magnitude-Frequency Distribution Consistency Test + # Evaluates consistency of the model MFD vs. observations using + # log-likelihood comparisons with stochastic catalogs. + M_test: + critical_frac: 0.025 # Fraction of simulations below which the + # test fails (e.g. 0.025 = 2.5th percentile) + prospective: False # [optional] Use prospective catalog + n_iters: 1000 # Number of Monte Carlo iterations + normalize_n_eqs: False # [optional] Normalize by number of + # earthquakes; default True + not_modeled_likelihood: 1.0e-5 # [optional] Likelihood for unmodeled + # bins; default 1e-5 + # investigation_time: 40. # [optional] Required if prospective: True + + # S-Test: Spatial Consistency Test + # Evaluates spatial consistency of the model by comparing per-cell + # likelihoods of observed vs. stochastic catalogs. + S_test: + critical_frac: 0.25 # Fraction threshold for test failure + prospective: False # [optional] Use prospective catalog + n_iters: 1000 # Number of Monte Carlo iterations + normalize_n_eqs: False # [optional] Normalize by number of + # earthquakes; default False + not_modeled_likelihood: 1.0e-5 # [optional] Likelihood for unmodeled + # cells; default 1e-5 + likelihood_function: mfd # [optional] Likelihood function to use; + # options: "mfd", "conf_interval_poisson" + # investigation_time: 40. # [optional] Required if prospective: True + + # L-Test: Likelihood Test + # Joint likelihood test combining spatial and magnitude information. + L_test: + critical_frac: 0.025 # Fraction threshold for test failure + prospective: False # [optional] Use prospective catalog + n_iters: 1000 # Number of Monte Carlo iterations + not_modeled_likelihood: 1.0e-5 # [optional] default 1e-5 + # investigation_time: 40. # [optional] Required if prospective: True + + # Moment Over/Under Evaluation + # Compares observed vs. stochastic moment release in each spatial cell. + moment_over_under: + investigation_time: 40. # Duration in years + n_iters: 100 # Number of stochastic event sets + # min_mag: 6.0 # [optional] Defaults to mfd_bin_min + # max_mag: 8.5 # [optional] Defaults to mfd_bin_max + + # Rupture Matching Evaluation + # Matches observed earthquakes to modeled ruptures based on location, + # magnitude, and (optionally) geometry. + rupture_matching_eval: + use_occurrence_rate: False # [optional] Weight by occurrence rate; + # default False + distance_lambda: 1.0 # [optional] Distance decay parameter + mag_window: 1.0 # [optional] Magnitude window for matching + group_return_threshold: 0.9 # [optional] Threshold for group matching + min_likelihood: 0.1 # [optional] Minimum match likelihood + no_attitude_default_like: 0.5 # [optional] Default likelihood when + # rupture has no attitude data + no_rake_default_like: 0.5 # [optional] Default likelihood when + # rupture has no rake data + return_one: best # [optional] "best" or "all" + parallel: False # [optional] Parallel matching; default False + + # Cumulative Occurrence Evaluation + # Evaluates cumulative earthquake occurrence over time by magnitude bin. + # Use {} for default parameters (no configuration needed). + cumulative_occurrence_eval: {} + + # Catalog Ground Motion Evaluation + # Compares observed ground motions from a flatfile with model predictions. + # Requires a flatfile to be specified under input. + catalog_ground_motion_eval: + match_rups: False # [optional] Match ruptures to earthquakes + # before ground motion comparison + # gmf_method: ground_motion_fields # [optional] Method for ground + # motion calculation + + + # ----------------------------------------------------------------------- + # RELM/CSEP Framework + # Implementation of the RELM (Regional Earthquake Likelihood Models) + # tests. These are similar to the GEM tests but use different statistical + # assumptions (e.g. not_modeled_likelihood is hardcoded to 0.0). + # ----------------------------------------------------------------------- + relm: + + N_test: + conf_interval: 0.95 + prob_model: poisson # "poisson" or "poisson_cum" + investigation_time: 40. + prospective: False # [optional] + + M_test: + critical_frac: 0.25 + n_iters: 1000 + investigation_time: 40. + prospective: False # [optional] + + S_test: + critical_frac: 0.25 + n_iters: 1000 + investigation_time: 40. + prospective: False # [optional] + likelihood_function: mfd # [optional] "mfd" or "conf_interval_poisson" + normalize_n_eqs: False # [optional] + + L_test: + critical_frac: 0.25 + n_iters: 1000 + investigation_time: 40. + prospective: False # [optional] + + + # ----------------------------------------------------------------------- + # Sanity Checks Framework + # Basic checks for model internal consistency. + # ----------------------------------------------------------------------- + sanity: + max_check: + warn: True # [optional] Log warnings; default True + + + + # ------------------------------------------------------------------------- + # Global config options + # ------------------------------------------------------------------------- + parallel: False # Use multiprocessing for source loading + # and intensive tests. Recommended for + # medium to large models. + + rand_seed: 69 # [optional] Random seed for reproducible + # Monte Carlo simulations. Must be an integer. + + log_file: hamlet_run.log # [optional] Path to log file output + + +# --------------------------------------------------------------------------- +# Inputs +# --------------------------------------------------------------------------- +input: + + # ------------------------------------------------------------------------- + # Spatial and magnitude binning + # ------------------------------------------------------------------------- + bins: + mfd_bin_min: 6.0 # Minimum magnitude (required) + mfd_bin_max: 9.0 # Maximum magnitude (required) + mfd_bin_width: 0.2 # Bin width (required) + h3_res: 3 # [optional] H3 hexagonal grid resolution + # (0-15, where 0 is coarsest); default 3 + + # ------------------------------------------------------------------------- + # Seismic Source Model (SSM) + # + # The SSM must be in OpenQuake format. Specify either ssm_dir + ssm_lt_file, + # or job_ini_file. + # ------------------------------------------------------------------------- + ssm: + ssm_dir: path/to/ssm/ # Directory containing the SSM files + ssm_lt_file: ssmLT.xml # Logic tree XML file name + # job_ini_file: job.ini # [optional] Alternative: specify an + # OpenQuake job.ini file instead of + # ssm_dir + ssm_lt_file. Default: null + + branch: b1 # [optional] Logic tree branch to evaluate. + # Set to "iterate" to evaluate each branch + # independently in a single run. Default: null + + tectonic_region_types: # [optional] Filter by tectonic region type. + - Active Shallow Crust # Must match types in the SSM. Default: null + # (all types included). + + source_types: null # [optional] Filter by source type. Default: null. Options: + # simple_fault, complex_fault, area, point, + # multipoint, MultiFaultSource. + # Pass as a list or null for all. + + # ------------------------------------------------------------------------- + # Depth filtering [optional] + # Sources outside this depth range are excluded from evaluation. + # ------------------------------------------------------------------------- + min_depth: 0 # [optional] Minimum source depth in km; default: 0.0 + max_depth: 40 # [optional] Maximum source depth in km; default: no limit + + # ------------------------------------------------------------------------- + # Rupture file caching [optional] + # Loading ruptures from a source model can be slow. These options let you + # save processed ruptures to disk and reload them in subsequent runs. + # ------------------------------------------------------------------------- + rupture_file: + read_rupture_file: false # Read ruptures from file instead of SSM; default: false + save_rupture_file: false # Save processed ruptures to file; default: false + rupture_file_path: ./ruptures.hdf5 # Path to the rupture file; default: null + # Supported formats: .hdf5, .feather, .csv + + # ------------------------------------------------------------------------- + # Spatial subsetting [optional] + # Restrict the evaluation to a geographic subset of the model domain. + # ------------------------------------------------------------------------- + subset: + file: path/to/subset.geojson # GIS file with subset geometry; default: null + buffer: 0.0 # Buffer distance around the subset geometry; default: 0.0 + + # ------------------------------------------------------------------------- + # Flatfile for ground motion evaluation [optional] + # Required for catalog_ground_motion_eval test. + # ------------------------------------------------------------------------- + flatfile: path/to/flatfile.csv + + # ------------------------------------------------------------------------- + # Simple ruptures [optional] + # Use simplified rupture representations (point sources) for faster + # processing. Default: true (from cfg_defaults). + # ------------------------------------------------------------------------- + simple_ruptures: true + + # ------------------------------------------------------------------------- + # Observed Earthquake Catalog + # ------------------------------------------------------------------------- + seis_catalog: + seis_catalog_file: path/to/catalog.csv + + # Temporal parameters: provide any two of start_date, stop_date, duration. + # The third will be calculated. Alternatively, use a completeness_table. + # start_date: 1976 # Can be an integer (year) or date string + stop_date: 2018-01-01 # Date string (YYYY-MM-DD) + # duration: 40. # Duration in years + + # Completeness table [optional] + # A list of [year, magnitude] pairs defining the completeness threshold + # over time. Each pair means "the catalog is complete above this + # magnitude from this year onward." When used, the effective duration + # varies by magnitude bin. + completeness_table: + - [1960, 5.0] + - [1900, 7.2] + + # Column mappings [optional] + # Map expected fields to actual column names in the CSV. Only specify + # columns whose names differ from the defaults. + columns: + # x_col: longitude # default: longitude + # y_col: latitude # default: latitude + # depth: depth # default: depth + # magnitude: magnitude # default: magnitude + # source: Agency # no default; institutional source of the eq + event_id: eventID # no default; earthquake ID column + + # Time column(s): either a single column name or a list of components + time: + - year + - month + - day + - hour + - minute + - second + + # ------------------------------------------------------------------------- + # Prospective catalog [optional] + # A separate catalog for prospective (forward-looking) evaluation. + # Uses the same format/columns as seis_catalog. + # ------------------------------------------------------------------------- + # prospective_catalog: + # seis_catalog_file: path/to/prospective_catalog.csv + # duration: 5. + + +# --------------------------------------------------------------------------- +# Report [optional] +# Generate an HTML report summarizing all test results. +# --------------------------------------------------------------------------- +report: + basic: + outfile: outputs/report.html + + +# --------------------------------------------------------------------------- +# JSON output [optional] +# Write test results as a JSON file. +# --------------------------------------------------------------------------- +json: + outfile: outputs/results.json diff --git a/doc_src/source/getting_started.md b/doc_src/source/getting_started.md index 82cda58..0aae682 100644 --- a/doc_src/source/getting_started.md +++ b/doc_src/source/getting_started.md @@ -5,9 +5,10 @@ steps to do this are: ## Prepare the hazard model -The hazard model must be in the OpenQuake format. There must be a single XML -file that describes the seismic source model logic tree, including the locations -of the source XML files and other logic tree parameters. +The hazard model must be in the OpenQuake format. There must be either a single +XML file that describes the seismic source model logic tree (including the +locations of the source XML files and other logic tree parameters), or an +OpenQuake `job.ini` file. ### Decide how the model should be evaluated @@ -16,18 +17,21 @@ different components of the model (different logic tree branches, different seismic source types, etc.). This will control how the data preparation and testing are done. +A single branch can be evaluated by specifying the `branch` parameter, or all +branches can be evaluated independently in one run by setting `branch: iterate`. + ### (Optional) Organize the hazard model with a `hamlet` directory It is recommended to add a new `hamlet` directory with `data` and `output` sub-directories. The `data` directory can hold the seismic catalogs and, if -necessary, GIS files specifying the model domain and grid cells. +necessary, GIS files specifying subdomains of the model. The `output` directory will hold the HTML reports and any other outputs that are written during the testing procedure. -The YAML configuration files can be placed in the main `hamlet` directory: +The [YAML configuration files](yaml_config_file.html) can be placed in the main `hamlet` directory: ``` model/ @@ -39,10 +43,11 @@ model/ data/ crustal_catalog.csv slab_catalog.csv - full_catalog.csv (unused in testing) + full_catalog.csv output/ test_ssm_crustal.yml test_ssm_slab.yml + test_ssm_all.yml ``` However, the user is free to organize the Hamlet files in any way; there is no @@ -56,7 +61,7 @@ This may be a good way of organizing the results and running Hamlet in a continuous integration system. For example, a `git` branch called `hamlet` can have a separate `hamlet` -directory, as specified above. This directory does not exist in the `master` +directory, as specified above. This directory does not exist in the `master` branch or other branches, and when changes are made to those branches, they can be pulled into the `hamlet` branch and be evaluated. @@ -67,16 +72,38 @@ necessarily when commits to `master` or development branches are made. ## Prepare the earthquake catalog(s) -The earthquake catalog should be declustered and, ideally, classified according -to the source types of the earthquakes (i.e., subduction thrust, in-slab, -crustal, etc.). The catalog should also be truncated to some acceptable -completeness date that corresponds to the `investigation_time` parameter used -during the Hamlet evaluations (in the future, completeness tables may be able to -be used instead of a single date, but this is not currently implemented). +The earthquake catalog should be declustered and, if desired, classified +according to the source types of the earthquakes (i.e., subduction thrust, +in-slab, crustal, etc.). The catalog(s) must be CSV files, with columns describing the fields and one row for each earthquake. +### Temporal completeness + +The temporal completeness of the catalog is very important for testing. In +general, a longer catalog will have a better sampling of events and therefore +be less noisy (i.e. a more consistent b-value). However before the 1970s, +catalogs in many areas do not have all earthquakes below, say, magnitude 7. +Therefore, it is desirable to understand the temporal completeness of the +catalog, so that you can only test against what is considered complete. + +The catalog's temporal completeness can be specified in one of two ways: + +1. **Simple date range**: Provide any two of `start_date`, `stop_date`, and + `duration` in the YAML configuration. The third will be calculated. This + assumes the catalog is complete above the minimum magnitude for the entire + time period. + +2. **Completeness table**: Provide a `completeness_table` as a list of + `[year, magnitude]` pairs. Each pair specifies that the catalog is complete + above that magnitude from that year onward. This allows Hamlet to account + for varying completeness across the magnitude range, giving more accurate + evaluations particularly for lower magnitudes with shorter complete periods. + + +### Separate catalogs for separate evaluations + If you are interested in running Hamlet separately for different seismic source types, then make separate catalogs for the different earthquake categories, i.e. make CSV files with only crustal events, only subduction megathrust @@ -89,14 +116,22 @@ should be split into separate files for each subset. ## Make the YAML configuration file(s) -See [YAML configuration file](./yaml_config_file.html) for more information. +See the [YAML configuration file documentation](yaml_config_file.html) for more information. ## Run Hamlet -Once the model, seismic catalog(s) and YAML configuration file(s) (and Hamlet -has been installed), Hamlet can be run like this: +Once the model, seismic catalog(s) and YAML configuration file(s) are prepared +(and Hamlet has been installed), Hamlet can be run like this: ``` hamlet test_ssm_crustal.yml ``` +The progress of the model processing and the evaluation results will be printed +to the terminal. However, the most useful and informative product is an html +file that has graphics for all the tests and some model information as well. + +Additional command-line options: + +- `hamlet --version` or `hamlet -v`: Print the version and exit. +- `hamlet config.yml --pdb`: Drop into the Python debugger on exception. diff --git a/doc_src/source/index.rst b/doc_src/source/index.rst index b2976d5..030b158 100644 --- a/doc_src/source/index.rst +++ b/doc_src/source/index.rst @@ -2,20 +2,18 @@ Hamlet: Hazard Model Evaluation and Testing ================================================== -Hamlet (``openquake.hme``) is a Python package developed (OK, in development) -for qualitative and quantitative evaluation of Probabilistic Seismic Hazard -Analysis (PSHA) models, with the intention of providing feedback to modelers -during the model construction process, to aid model development. Hamlet is -developed by the `GEM Foundation`_, and uses the OpenQuake_ software -extensively. - -Hamlet will incorporate several model test frameworks, including those -developed by GEM and some of those developed outside of GEM such as the RELM_ -tests. Currently, and likely in the future, the model files will be required to -be in the OpenQuake_ format, regardless of the format of their original -implementation. - -Most of the Hamlet evaluations are spatial in nature; the model domain is +Hamlet (``openquake.hme``) is a Python package for qualitative and quantitative +evaluation of Probabilistic Seismic Hazard Analysis (PSHA) models, with the +intention of providing feedback to modelers during the model construction +process, to aid model development. Hamlet is developed by the `GEM +Foundation`_, and uses the OpenQuake_ software extensively. + +Hamlet incorporates several test frameworks, including those developed by +GEM and some of those developed outside of GEM such as the RELM_/CSEP tests. +The model files must be in the OpenQuake_ format (or as a suitably-formatted +CSV of ruptures). + +Many of the Hamlet tests and evaluations are spatial in nature; the model domain is discretized into grid cells, and comparisons between observations and model predictions are performed in each grid cell, to highlight where in the domain the model matches the observations, and where it might need some refinement. @@ -27,26 +25,21 @@ branch of a source model logic tree can be tested independently, and each type of source (e.g., subduction megathrust, crustal, in-slab) can be tested independently as well, in the spatial framework described above. +A quick note on terminology: We use the word 'test' for an analysis that has a +pass/fail outcome tied to a criterion, and 'evaluation' for an analysis that +provides some quantitative or qualitative information on model performance but +is not linked to a pass/fail outcome. + Quickstart ========== Installation ------------ -Hamlet requires installation Python v.3.7+, the OpenQuake_ engine, and some -additional dependencies as well. These are specified in the ``requirements.txt`` -file. - -(*Note:* A few of the dependencies might be challenging to install. These are -Rtree_ and h3-py_. You may have to install ``libspatialindex`` or -``libspatialindex-dev`` on Linux or MacOS first, depending on your system, for -``Rtree``. ``h3-py`` requires ``cc`` and ``make``, but then on Linux/MacOS can -be installed easily. Please see the documentation for each.) +Hamlet requires Python 3.11+ and the OpenQuake_ engine. First, install the OpenQuake_ engine, following directions on that website. You -probably want to install it into a virtual environment, and you may even want to -have a separate virtual environment for running Hamlet than the OpenQuake_ -virtual environment that you normally use (this is up to you). +probably want to install it into a virtual environment. Then, clone the Hamlet repository, and from that directory, install the requirements:: @@ -61,14 +54,14 @@ and then install Hamlet:: Running Hamlet -------------- -Hamlet only requires a seismic hazard model (implemented in OpenQuake_) and a +Hamlet requires a seismic hazard model (implemented in OpenQuake_) and a processed seismic catalog (declustered, and ideally classified by source type) to run. Once installed, Hamlet can be run from the command line:: hamlet test_model.yml ``test_model.yml`` is a :doc:`configuration file ` in YAML_ -format that specifies the source model, seismic catalog, tests to be run, and +format that specifies the source model, seismic catalog, tests and evaluations to be run, and other variables and parameters. @@ -79,8 +72,10 @@ Documentation :caption: Contents: getting_started + yaml_config_file architecture model_test_frameworks/model_test_frameworks + api @@ -97,4 +92,3 @@ Indices and tables .. _OpenQuake: https://github.com/GEM/oq-engine .. _RELM: http://cseptesting.org/documents/relm.php .. _h3-py: https://github.com/uber/h3-py -.. _Rtree: https://toblerity.org/rtree/ diff --git a/doc_src/source/model_test_frameworks/gem_tests.rst b/doc_src/source/model_test_frameworks/gem_tests.rst index 275a027..c0c7c16 100644 --- a/doc_src/source/model_test_frameworks/gem_tests.rst +++ b/doc_src/source/model_test_frameworks/gem_tests.rst @@ -4,119 +4,293 @@ GEM Tests and Evaluations ========================= +These tests are developed by GEM, some based on the literature (e.g. Zechar et +al. 2010), some based on GEM's own ideas and implementations. See +:mod:`~openquake.hme.model_test_frameworks.gem` for the function documentation. +Sanity checks (as detailed below) are also available from the GEM testing +framework, for convenience during the workflow. -These are developed by GEM, sometimes based on the literature, or sometimes -based on our own ideas and implementations. See for -more information, or :mod:`~openquake.hme.model_test_frameworks.gem` for the -function documentation. Sanity checks (as detailed below) are also available -from the GEM testing framework, for convenience during the workflow. +The GEM framework includes the N, M, S, and L consistency tests (similar to the +RELM/CSEP versions but with configurable handling of unmodeled cells/bins), as +well as several additional evaluations: MFD comparison, moment rate analysis, +rupture matching, cumulative occurrence, and ground motion evaluation. -Magnitude-Frequency Distribution based tests --------------------------------------------- +.. note:: -These tests evaluate a model based on the magnitude-frequency distribution (MFD) -inside each grid cell, or in the model as a whole, (for the model component -being tested). + The ``likelihood`` test is deprecated and should not be used. Use the + ``M_test``, ``S_test``, and ``L_test`` instead, which provide more robust + and well-characterized likelihood-based consistency tests. -.. _gem-like-test: +Statistical Consistency Tests +----------------------------- -*Likelihood test* +These tests evaluate model consistency using Monte Carlo simulations. Each test +generates many stochastic catalogs from the model and compares a test statistic +of the observed catalog against the distribution of the stochastic catalogs. -Currently, there are two implementations of the MFD likelihood tests, one -'empirical', based on Monte Carlo sampling of the source model in each bin, and -one based on the frequencies in the MFD itself. -In both of these tests, the likelihood of observing the seismicity in the -catalog given the model MFD is calculated through the +.. _gem-N-test: -Parameters (all are optional, as default values are supplied): +N-Test +~~~~~~ -``likelihood_method`` - This is how the computations are performed. ``poisson`` uses the Poisson - likelihood, and ``empirical`` uses a Monte Carlo sampling of the MFD. The - default value is ``poisson``. +Compares the total number of observed earthquakes to the number predicted by the +model. The observed count is checked against a confidence interval derived from +the Poisson (or cumulative Poisson) distribution. -``investigation_time`` - This is how long the time period is for comparing the observed seismicity to - the MFD. Unless you're doing something very crafty, the value should be in - years, and should be the length of the observed earthquake catalog. In the - future, a completeness table may be used instead of this parameter. The - default value is ``1.`` +Parameters: + +``conf_interval`` + Confidence interval for the test (e.g. ``0.95`` means 95%). + +``prob_model`` + Probability model: ``"poisson"`` or ``"poisson_cum"``. + +``prospective`` + Optional. If ``True``, use the prospective catalog instead of the + retrospective catalog. Default: ``False``. + + +.. _gem-M-test: + +M-Test +~~~~~~ + +Evaluates the consistency of the magnitude-frequency distribution of the model +vs. the observations. The log-likelihood of the observed earthquakes given the +model forecast is compared with the log-likelihoods of stochastic catalogs +generated from the same forecast. If the observed log-likelihood falls below the +``critical_frac`` threshold of the stochastic distribution, the test fails. + +The log-likelihoods are calculated for each magnitude bin using the Poisson +distribution, then aggregated as the geometric mean across bins. + +This test is based on Zechar et al. (2010) with two differences: (1) the total +number of earthquakes in stochastic simulations is not fixed, and (2) the +geometric mean is used instead of the product of bin likelihoods. Neither +difference affects pass/fail outcomes meaningfully. + +Parameters: + +``critical_frac`` + Fraction of simulations below which the test fails (e.g. ``0.025`` + for a 2.5th percentile threshold). + +``n_iters`` + Number of Monte Carlo iterations. + +``prospective`` + Optional. Default: ``False``. + +``normalize_n_eqs`` + Optional. Normalize the number of earthquakes in stochastic catalogs + to match the observed count. Default: ``True``. + +``not_modeled_likelihood`` + Optional. Likelihood assigned to magnitude bins with zero modeled rate + but observed earthquakes. Default: ``1e-5``. -``default_likelihood`` - This is the likelihood that results if no earthquake sources are present in - the grid cell. If the cells are built using `h3` (the default option, if no - GIS file for the test is supplied), this parameter will have no effect. The - default value is ``1.`` +.. _gem-S-test: + +S-Test +~~~~~~ + +Evaluates the spatial consistency of the model by comparing per-cell likelihoods +of the observed catalog against stochastic catalogs. This highlights spatial +cells where the model over- or under-predicts seismicity. + +Parameters: + +``critical_frac`` + Fraction threshold for test failure. + +``n_iters`` + Number of Monte Carlo iterations. + +``prospective`` + Optional. Default: ``False``. + +``normalize_n_eqs`` + Optional. Normalize by number of earthquakes. Default: ``False``. + +``not_modeled_likelihood`` + Optional. Default: ``1e-5``. + +``likelihood_function`` + Optional. The likelihood function to use for per-cell evaluation. Options: + ``"mfd"`` (default) or ``"conf_interval_poisson"``. + + +.. _gem-L-test: + +L-Test +~~~~~~ + +Joint likelihood test combining spatial and magnitude information. This is the +most comprehensive consistency test, evaluating the overall likelihood of the +observed catalog given the model. + +Parameters: + +``critical_frac`` + Fraction threshold for test failure. -``not_modeled_val`` - This is the likelihood that results if the rate of earthquake production in - that magnitude bin is zero, but there are earthquakes within the magnitude - bin. Standard (or naive) statistical theory suggests that this value should - be zero, as it is in the RELM tests, but because this value is multiplied by - all of the other magnitude bins inside each spatial bin/grid cell, a single - zero value will make the whole model likelihood zero. The default is - ``1e-5`` which is a bit more pragmatic. +``n_iters`` + Number of Monte Carlo iterations. +``prospective`` + Optional. Default: ``False``. +``not_modeled_likelihood`` + Optional. Default: ``1e-5``. -.. _gem-model-mfd-test: -*Model MFD Evaluation* +Magnitude-Frequency Distribution Evaluations +--------------------------------------------- -The Model MFD evaluation sums up the MFDs from each -:class:`~openquake.hme.utils.bins.SpacemagBin` and makes an MFD for the whole -model, which is then compared to the observed MFD from the earthquake catalog. -This is to produce a figure and currently does not yield any quantitative values -or evaluate the entire model goodness of fit. +.. _gem-model-mfd-eval: -Parameters (all are optional): +Model MFD Evaluation (``model_mfd``) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Sums up the MFDs from all spatial cells to produce a total model MFD, which is +compared to the observed MFD from the earthquake catalog. This produces a figure +in the report showing both MFDs. Use ``{}`` for default parameters. + +Parameters (all optional): ``investigation_time`` - This is the duration of the comparison between the observed seismicity to - the MFD. See :ref:`likelihood ` above for more information. + Duration in years. Defaults to the seismic catalog duration. + +``annualize`` + If ``True``, annualize the rates. Default: ``True``. -``out_csv`` - This parameter specifies a filename. If this is given, a CSV table of the - observed and modeled MFDs will be written. -``out_plot`` - This parameter specifies a filename. If this is given, a plot of the - observed and modeled MFDs will be written. The file suffix will determine - the plot format. Common formats include ``png``, ``svg`` and ``pdf``. See - the ``matplotlib`` docs for more info. +.. _gem-max-mag-check: +Maximum Magnitude Check (``max_mag_check``) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -.._max_mag_check: +A sanity check that verifies the model can produce earthquakes as large as the +largest observed earthquake in each spatial cell. Note that there can be issues +with very large earthquakes (with ruptures larger than the cell size), as the +hypocenter for an observed event may be in a different cell than the most +compatible source. + +Parameters: -*Ensures that the model can produce the maximum observed seismcity in each cell* +``append_check`` + Optional. Boolean. If ``True``, append pass/fail results to the bin data. -This test is borrowed from the Sanity checks. It simply checks to see whether -the sources inside each cell are capable of producing earthquakes as large as -the largest observed earthquakes. Note that there can be some issues with very -large earthquakes (with ruptures larger than the cell size) as the hypocenter -for an observed event may be in a different cell than the most compatible -hypocenter from the sources. +``warn`` + Optional. Boolean. If ``True``, log warnings for each failing cell. +Other Evaluations +----------------- + .. _gem-moment-over-under-eval: -*Compares observed and stochastic moment release* +Moment Over/Under Evaluation (``moment_over_under``) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -This evaluation generates many synthetic catalogs (stochastic event sets) and -compares the total moment release in each cell for each of the catalogs to the -observed moment release. This evalution helps highlight areas that are more or -less seismically productive than the observations may support. +Generates many stochastic catalogs and compares the total seismic moment release +in each spatial cell to the observed moment release. This helps highlight areas +that are more or less seismically productive than the observations support. Parameters: -``investigation_time``: Duration of the catalog (and of the generated - stochastic event sets). +``investigation_time`` + Duration of the catalog in years. + +``n_iters`` + Number of stochastic event sets to generate. + +``min_mag`` + Optional. Minimum magnitude for moment calculation. Defaults to + ``mfd_bin_min``. + +``max_mag`` + Optional. Maximum magnitude for moment calculation. Defaults to + ``mfd_bin_max``. + + +.. _gem-rupture-matching-eval: + +Rupture Matching Evaluation (``rupture_matching_eval``) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Matches observed earthquakes to modeled ruptures based on proximity, magnitude +similarity, and (optionally) geometric similarity (attitude and rake). This +evaluation helps assess whether the model contains ruptures that are consistent +with the observed earthquakes. + +Parameters (all optional, with defaults): + +``use_occurrence_rate`` + Weight matches by rupture occurrence rate. Default: ``False``. + +``distance_lambda`` + Distance decay parameter for the matching function. Default: ``1.0``. -``n_iters``: Number of iterations (stochastic event sets) generated. Note that - generating these catalogs is fairly time-intensive in the current - implementation. For large models, the number of iterations should be kept to - under 50-100 until performance is improved. +``mag_window`` + Magnitude window for considering candidate ruptures. Default: ``1.0``. + +``group_return_threshold`` + Threshold for group matching. Default: ``0.9``. + +``min_likelihood`` + Minimum match likelihood. Default: ``0.1``. + +``no_attitude_default_like`` + Default likelihood when a rupture has no attitude data. Default: ``0.5``. + +``no_rake_default_like`` + Default likelihood when a rupture has no rake data. Default: ``0.5``. + +``return_one`` + ``"best"`` to return only the best match, or ``"all"`` for all matches + above the threshold. Default: ``"best"``. + +``parallel`` + Use parallel processing for matching. Default: ``False``. + + +.. _gem-cumulative-occurrence-eval: + +Cumulative Occurrence Evaluation (``cumulative_occurrence_eval``) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Evaluates the cumulative earthquake occurrence over time for each magnitude bin, +comparing the observed temporal pattern to the model's predicted rate. This is +useful for identifying temporal clustering or quiescence relative to the model. + +Takes no configuration parameters (use ``{}``). + + +.. _gem-catalog-ground-motion-eval: + +Catalog Ground Motion Evaluation (``catalog_ground_motion_eval``) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Compares observed ground motions from a flatfile with model predictions. This +requires a flatfile to be specified under ``input.flatfile`` in the +configuration. + +Parameters: +``match_rups`` + Optional. If ``True``, match earthquakes to model ruptures before computing + ground motion comparisons; if ``False``, then new ruptures are generated + based on the earthquake information in the flatfile. If ruptures are + matched, it provides a better understanding of whether the model is + reproducing the ground motions from specific earthquakes, and the + confidence with which an earthquake is assigned to a tectonic region type + is much higher (as the best-matching rupture will already have one + defined). Default: ``False``. + +``gmf_method`` + Optional. Method for ground motion calculation. Default: + ``"ground_motion_fields"``. diff --git a/doc_src/source/model_test_frameworks/model_test_frameworks.rst b/doc_src/source/model_test_frameworks/model_test_frameworks.rst index 016354d..6b67084 100644 --- a/doc_src/source/model_test_frameworks/model_test_frameworks.rst +++ b/doc_src/source/model_test_frameworks/model_test_frameworks.rst @@ -2,68 +2,148 @@ Model Testing Frameworks ======================== +Hamlet provides four frameworks that can be used independently or combined +in a single Hamlet run. Each framework is specified as a key under +``config.model_framework`` in the YAML configuration file. + .. include:: ./gem_tests.rst -Sanity Checks -============= -Sanity checks are small evaluations to make sure that basic parameters of the -model are internally consistent and match the observations at a gross level. An -example is making sure that the MFDs in each grid cell are capable of producing -earthquakes as large as the largest observed earthquakes. +.. _relm-tests: -Tests ------ +RELM/CSEP Tests +================ -.. _max-mag-check: +The RELM_ (Regional Earthquake Likelihood Models) / CSEP tests are +implementations of the standard CSEP consistency tests. These are similar to the +GEM tests but differ in some statistical assumptions -- notably, +``not_modeled_likelihood`` is hardcoded to ``0.0`` in the RELM framework, +meaning that any observations in unmodeled cells or magnitude bins will cause +the test to fail. + +Available tests: + +.. _relm-N-test: -*Maximum Magnitude Check* +N-Test +------ -The Maximum Magnitude Check evaluates the observed seismicity and MFD inside -each class:`~openquake.hme.utils.bins.SpacemagBin` to see whether the maximum -magnitude of the MFD is larger than the observed earthquakes. +Compares the total number of observed earthquakes to the model prediction. +See :ref:`gem-N-test` for details on the test logic. Parameters: -``append_check`` - This is a Boolean (``True`` or ``False``). If ``True``, the row for each - :class:`~openquake.hme.utils.bins.SpacemagBin` in the dataframe used to - store everything will have a column appended that gives ``True`` and - ``False`` values for if the test passes or not (``True`` is pass). +``conf_interval`` + Confidence interval for the test (e.g. ``0.95``). -``warn`` - This is a Boolean (``True`` or ``False``) that determines whether a - warning is given on the command line (or in the log file) for each - class:`~openquake.hme.utils.bins.SpacemagBin` that fails the test. +``prob_model`` + Probability model: ``"poisson"`` or ``"poisson_cum"``. + +``investigation_time`` + Duration in years. + +``prospective`` + Optional. Use prospective catalog. Default: ``False``. + + +.. _relm-M-test: + +M-Test +------ + +Evaluates the consistency of the magnitude-frequency distribution. +See :ref:`gem-M-test` for details on the test logic. + +Parameters: + +``critical_frac`` + Fraction threshold for test failure (e.g. ``0.25``). + +``n_iters`` + Number of Monte Carlo iterations. + +``investigation_time`` + Duration in years. + +``prospective`` + Optional. Use prospective catalog. Default: ``False``. + + +.. _relm-S-test: +S-Test +------ +Evaluates the spatial consistency of the model. +See :ref:`gem-S-test` for details on the test logic. -RELM Tests -========== +Parameters: + +``critical_frac`` + Fraction threshold for test failure. + +``n_iters`` + Number of Monte Carlo iterations. + +``investigation_time`` + Duration in years. + +``prospective`` + Optional. Default: ``False``. + +``likelihood_function`` + Optional. ``"mfd"`` or ``"conf_interval_poisson"``. Default: ``"mfd"``. -The RELM_ (Regional Earthquake Likelihood Model) tests are quite similar in -principle to the GEM Magnitude-Frequency evaluations, though there are some -differences in the statistical assumptions used to aggregate the likelihood -statistics from model sub-regions to the model as a whole. +``normalize_n_eqs`` + Optional. Normalize by number of earthquakes. Default: ``False``. -.. _N_test: -*N test* +.. _relm-L-test: - The N-test compares the total number of events in the catalog to the total - number of events in stochastic events sets generated from the source model. - This is the count of the lowest-magnitude bin or point in the - magnitude-frequency distribution. +L-Test +------ + +Joint likelihood test combining spatial and magnitude information. Parameters: +``critical_frac`` + Fraction threshold for test failure. + +``n_iters`` + Number of Monte Carlo iterations. + +``investigation_time`` + Duration in years. -.. _M_test: +``prospective`` + Optional. Default: ``False``. -*M test* -.. _S_test: +.. _sanity-checks: + +Sanity Checks +============= + +Sanity checks are basic tests to verify that the model is internally +consistent and matches the observations at a gross level. + +.. _max-mag-check: + +Maximum Magnitude Check (``max_check``) +--------------------------------------- + +Evaluates the observed seismicity and MFD inside each spatial cell to check +whether the maximum magnitude of the MFD is larger than the largest observed +earthquake. Cells where the observed maximum magnitude exceeds the model +maximum magnitude are flagged. + +Parameters: + +``warn`` + Optional. Boolean. If ``True``, log a warning for each cell that fails + the check. Default: ``True``. -.. _RELM: http://cseptesting.org/documents/relm.php \ No newline at end of file +.. _RELM: http://cseptesting.org/documents/relm.php diff --git a/doc_src/source/modules.rst b/doc_src/source/modules.rst deleted file mode 100644 index c04cf83..0000000 --- a/doc_src/source/modules.rst +++ /dev/null @@ -1,7 +0,0 @@ -openquake -========= - -.. toctree:: - :maxdepth: 4 - - openquake diff --git a/doc_src/source/openquake.hme.core.rst b/doc_src/source/openquake.hme.core.rst deleted file mode 100644 index 81f43c9..0000000 --- a/doc_src/source/openquake.hme.core.rst +++ /dev/null @@ -1,22 +0,0 @@ -openquake.hme.core package -========================== - -Submodules ----------- - -openquake.hme.core.core module ------------------------------- - -.. automodule:: openquake.hme.core.core - :members: - :undoc-members: - :show-inheritance: - - -Module contents ---------------- - -.. automodule:: openquake.hme.core - :members: - :undoc-members: - :show-inheritance: diff --git a/doc_src/source/openquake.hme.model_test_frameworks.gem.rst b/doc_src/source/openquake.hme.model_test_frameworks.gem.rst deleted file mode 100644 index 38ac652..0000000 --- a/doc_src/source/openquake.hme.model_test_frameworks.gem.rst +++ /dev/null @@ -1,38 +0,0 @@ -openquake.hme.model\_test\_frameworks.gem package -================================================= - -Submodules ----------- - -openquake.hme.model\_test\_frameworks.gem.gem\_stats module ------------------------------------------------------------ - -.. automodule:: openquake.hme.model_test_frameworks.gem.gem_stats - :members: - :undoc-members: - :show-inheritance: - -openquake.hme.model\_test\_frameworks.gem.gem\_test\_functions module ---------------------------------------------------------------------- - -.. automodule:: openquake.hme.model_test_frameworks.gem.gem_test_functions - :members: - :undoc-members: - :show-inheritance: - -openquake.hme.model\_test\_frameworks.gem.gem\_tests module ------------------------------------------------------------ - -.. automodule:: openquake.hme.model_test_frameworks.gem.gem_tests - :members: - :undoc-members: - :show-inheritance: - - -Module contents ---------------- - -.. automodule:: openquake.hme.model_test_frameworks.gem - :members: - :undoc-members: - :show-inheritance: diff --git a/doc_src/source/openquake.hme.model_test_frameworks.relm.rst b/doc_src/source/openquake.hme.model_test_frameworks.relm.rst deleted file mode 100644 index 7c001f9..0000000 --- a/doc_src/source/openquake.hme.model_test_frameworks.relm.rst +++ /dev/null @@ -1,30 +0,0 @@ -openquake.hme.model\_test\_frameworks.relm package -================================================== - -Submodules ----------- - -openquake.hme.model\_test\_frameworks.relm.relm\_stats module -------------------------------------------------------------- - -.. automodule:: openquake.hme.model_test_frameworks.relm.relm_stats - :members: - :undoc-members: - :show-inheritance: - -openquake.hme.model\_test\_frameworks.relm.relm\_tests module -------------------------------------------------------------- - -.. automodule:: openquake.hme.model_test_frameworks.relm.relm_tests - :members: - :undoc-members: - :show-inheritance: - - -Module contents ---------------- - -.. automodule:: openquake.hme.model_test_frameworks.relm - :members: - :undoc-members: - :show-inheritance: diff --git a/doc_src/source/openquake.hme.model_test_frameworks.rst b/doc_src/source/openquake.hme.model_test_frameworks.rst deleted file mode 100644 index 6888250..0000000 --- a/doc_src/source/openquake.hme.model_test_frameworks.rst +++ /dev/null @@ -1,19 +0,0 @@ -openquake.hme.model\_test\_frameworks package -============================================= - -Subpackages ------------ - -.. toctree:: - - openquake.hme.model_test_frameworks.gem - openquake.hme.model_test_frameworks.relm - openquake.hme.model_test_frameworks.sanity - -Module contents ---------------- - -.. automodule:: openquake.hme.model_test_frameworks - :members: - :undoc-members: - :show-inheritance: diff --git a/doc_src/source/openquake.hme.model_test_frameworks.sanity.rst b/doc_src/source/openquake.hme.model_test_frameworks.sanity.rst deleted file mode 100644 index 4384a5d..0000000 --- a/doc_src/source/openquake.hme.model_test_frameworks.sanity.rst +++ /dev/null @@ -1,30 +0,0 @@ -openquake.hme.model\_test\_frameworks.sanity package -==================================================== - -Submodules ----------- - -openquake.hme.model\_test\_frameworks.sanity.sanity\_checks module ------------------------------------------------------------------- - -.. automodule:: openquake.hme.model_test_frameworks.sanity.sanity_checks - :members: - :undoc-members: - :show-inheritance: - -openquake.hme.model\_test\_frameworks.sanity.sanity\_test\_functions module ---------------------------------------------------------------------------- - -.. automodule:: openquake.hme.model_test_frameworks.sanity.sanity_test_functions - :members: - :undoc-members: - :show-inheritance: - - -Module contents ---------------- - -.. automodule:: openquake.hme.model_test_frameworks.sanity - :members: - :undoc-members: - :show-inheritance: diff --git a/doc_src/source/openquake.hme.reporting.rst b/doc_src/source/openquake.hme.reporting.rst deleted file mode 100644 index fa31944..0000000 --- a/doc_src/source/openquake.hme.reporting.rst +++ /dev/null @@ -1,22 +0,0 @@ -openquake.hme.reporting package -=============================== - -Submodules ----------- - -openquake.hme.reporting.reporting module ----------------------------------------- - -.. automodule:: openquake.hme.reporting.reporting - :members: - :undoc-members: - :show-inheritance: - - -Module contents ---------------- - -.. automodule:: openquake.hme.reporting - :members: - :undoc-members: - :show-inheritance: diff --git a/doc_src/source/openquake.hme.rst b/doc_src/source/openquake.hme.rst deleted file mode 100644 index 42cb582..0000000 --- a/doc_src/source/openquake.hme.rst +++ /dev/null @@ -1,20 +0,0 @@ -openquake.hme package -===================== - -Subpackages ------------ - -.. toctree:: - - openquake.hme.core - openquake.hme.model_test_frameworks - openquake.hme.reporting - openquake.hme.utils - -Module contents ---------------- - -.. automodule:: openquake.hme - :members: - :undoc-members: - :show-inheritance: diff --git a/doc_src/source/openquake.hme.utils.rst b/doc_src/source/openquake.hme.utils.rst deleted file mode 100644 index df42591..0000000 --- a/doc_src/source/openquake.hme.utils.rst +++ /dev/null @@ -1,78 +0,0 @@ -openquake.hme.utils package -=========================== - -Submodules ----------- - -openquake.hme.utils.bins module -------------------------------- - -.. automodule:: openquake.hme.utils.bins - :members: - :undoc-members: - :show-inheritance: - -openquake.hme.utils.io module ------------------------------ - -.. automodule:: openquake.hme.utils.io - :members: - :undoc-members: - :show-inheritance: - -openquake.hme.utils.model module --------------------------------- - -.. automodule:: openquake.hme.utils.model - :members: - :undoc-members: - :show-inheritance: - -openquake.hme.utils.moment\_tensor module ------------------------------------------ - -.. automodule:: openquake.hme.utils.moment_tensor - :members: - :undoc-members: - :show-inheritance: - -openquake.hme.utils.plots module --------------------------------- - -.. automodule:: openquake.hme.utils.plots - :members: - :undoc-members: - :show-inheritance: - -openquake.hme.utils.simple\_rupture module ------------------------------------------- - -.. automodule:: openquake.hme.utils.simple_rupture - :members: - :undoc-members: - :show-inheritance: - -openquake.hme.utils.stats module --------------------------------- - -.. automodule:: openquake.hme.utils.stats - :members: - :undoc-members: - :show-inheritance: - -openquake.hme.utils.utils module --------------------------------- - -.. automodule:: openquake.hme.utils.utils - :members: - :undoc-members: - :show-inheritance: - - -Module contents ---------------- - -.. automodule:: openquake.hme.utils - :members: - :undoc-members: - :show-inheritance: diff --git a/doc_src/source/openquake.rst b/doc_src/source/openquake.rst deleted file mode 100644 index 5c72482..0000000 --- a/doc_src/source/openquake.rst +++ /dev/null @@ -1,17 +0,0 @@ -openquake package -================= - -Subpackages ------------ - -.. toctree:: - - openquake.hme - -Module contents ---------------- - -.. automodule:: openquake - :members: - :undoc-members: - :show-inheritance: diff --git a/doc_src/source/yaml_config_file.rst b/doc_src/source/yaml_config_file.rst index 0052c79..2833309 100644 --- a/doc_src/source/yaml_config_file.rst +++ b/doc_src/source/yaml_config_file.rst @@ -5,21 +5,16 @@ The YAML configuration file =========================== -A Hamlet evaluation is configured using a `YAML `_ file, -which describes the tests that should be run and any required parameters, the -location of the input files, and what outputs should be written. The file is -made into a dictionary that is used guide the Hamlet run. Therefore the order +A Hamlet evaluation is configured using a `YAML `_ file, +which describes the tests and evaluations that should be run and any required parameters, the +location of the input files, and what outputs should be written. The file is +made into a dictionary that is used to guide the Hamlet run, so the order of items is not important. -If different components of the model are to be evaluated (different logic tree -branches, or different source model types, etc.) then a separate YAML file -should be made for each logic tree branch and source model combination to be -evaluated, as this will be done in separate Hamlet runs. - -A simple example of a yaml configuration file is given at the bottom of this -page. - -Here are the components of the YAML config file: +A simple example of a YAML configuration file is given at the bottom of this +page, and a comprehensive reference configuration (with comments explaining +every option) is available in the source repository at +``doc_src/source/example_configs/reference_config.yml``. .. _metadata: @@ -27,14 +22,14 @@ Here are the components of the YAML config file: Metadata (``meta``) =================== -This is a basic section that describes the model. The information is used when -writing reports; otherwise it is not necessary. +This optional section describes the model. The information is used when +writing reports. .. code-block:: yaml meta: model_name: Model Name - description: "words describing the model name" + description: "words describing the model" @@ -43,69 +38,71 @@ writing reports; otherwise it is not necessary. Test and Evaluation Configuration (``config``) ============================================== -This section lists the different tests or evaluations to be run, and what the -parameters for each will be. +This section specifies which frameworks, tests, and evaluations to run, along +with their parameters. ``model_framework`` - This specifies which framework to use (see - :doc:`./model_test_frameworks/model_test_frameworks` for more information on - these). Currently, only the ``gem`` option is available, but ``sanity`` and - ``relm`` are close to being usable. However the ``sanity`` checks are - available through the ``gem`` framework as well. +------------------- -``parallel`` - This is a Boolean (``True`` or ``False``) flag that determines whether - parallel algorithms are used for loading the seismic source model, and - performing the more time-intensive tests (such as Monte Carlo based tests). - The parallel algorithms produce equivalent results, using Python's - `multiprocessing `_ - facilities. +This is a dictionary of frameworks to run. Each framework key contains +its own tests and evaluations as sub-keys. Multiple frameworks can be run in a +single Hamlet run. Available frameworks are: - This flag should be ``True`` for medium to large models, unless RAM is a - major limitation. For small models, it may be faster to run on a single - core, because the overhead in instantiating multiple processes, and - processing the results, can be substantial. +* ``gem`` -- GEM tests and evaluations (see :ref:`gem-tests-evaluations`). The + most supported and/or configurable versions of all tests and evaluations are + available here. It is recommended to use these. +* ``relm`` -- RELM/CSEP tests (see :ref:`relm-tests`). These 'letter tests' are + found with specific defaults set. +* ``sanity`` -- Basic sanity checks (see :ref:`sanity-checks`). -``rand_seed`` - A `random seed`_ to be used for reproducible Monte Carlo simulations and - other functions using random sampling. The seed must follow Numpy's rules; - to keep it simple, just use an integer. Optional. +Tests and evaluations are nested directly under their framework, with their +configuration parameters as sub-keys. Use an empty dictionary (``{}``) for +those that need no configuration. -.. _random seed: https://docs.scipy.org/doc/numpy-1.15.1/reference/generated/numpy.random.seed.html +.. code-block:: yaml -Tests ------ + config: + model_framework: + gem: + model_mfd: {} + max_mag_check: + append_check: True + warn: True + N_test: + conf_interval: 0.95 + prob_model: poisson + M_test: + critical_frac: 0.025 + n_iters: 1000 + relm: + S_test: + critical_frac: 0.25 + n_iters: 1000 + investigation_time: 40. -``tests`` - This specifies which tests or evaluations within the - ``model_test_framework`` should be run. Multiple tests can be run; these - should be nested below the ``tests`` item with their configuration variables - given. - For the ``gem`` framework, the available tests are - :ref:`likelihood `, - :ref:`model_mfd `, and - :ref:`max_mag_check `. +``parallel`` + A Boolean (``True`` or ``False``) flag that determines whether + parallel algorithms are used for loading the seismic source model and + performing the more time-intensive tests and evaluations (such as Monte + Carlo based consistency tests and moment analysis). + This flag should be ``True`` for medium to large models, unless RAM is a + major limitation. For small models, it may be faster to run on a single + core, because the overhead in instantiating multiple processes can be + substantial. -An example, illustrating the required nesting: +``rand_seed`` + An optional integer `random seed`_ for reproducible random sampling (which + is fundamental in Hamlet, as stochastic catalogs are generated for most + tests and evaluations). -.. code-block:: yaml +``log_file`` + An optional path to a log file. If given, log output is written to this + file in addition to the console. - config: - model_framework: gem - tests: - likelihood: - likelihood_method: poisson - investigation_time: 40. - model_mfd: - investigation_time: 40. - max_mag_check: - append_check: True - warn: True - parallel: False - rand_seed: 420 +.. _random seed: https://docs.scipy.org/doc/numpy-1.15.1/reference/generated/numpy.random.seed.html .. _input: @@ -114,232 +111,285 @@ Inputs (``input``) ================== This section describes the inputs into a Hamlet run: the seismic source model, -the observed earthquake catalog that is used to evaluate the seismic source -model, and the configuration of the spatial bins that will hold the SSM. +the observed earthquake catalog, and the configuration of the spatial bins. Seismic Source Model (``ssm``) ------------------------------ -This section describes the location of the Seismic Souce Model (SSM), and the -components of the model that should be evaluated with Hamlet. - -The SSM must be in the modern OpenQuake format, and specified with a logic tree -XML file. +The SSM must be in the modern OpenQuake format. It can be specified either with +a logic tree XML file or an OpenQuake job.ini file. All SSM parameters default +to ``null`` when not specified (see ``cfg_defaults`` in +``openquake.hme.core.core``). Sub-parameters: -* ``ssm_dir`` - This parameter describes the filepath (either absolute or relative) to the - directory containing the seismic source model logic tree file. +``ssm_dir`` + Filepath (absolute or relative) to the directory containing the seismic + source model logic tree file. -* ``ssm_lt_file`` - This is the name of the logic tree XML file for the SSM, i.e. ``ssmLT.xml`` +``ssm_lt_file`` + The name of the logic tree XML file for the SSM, i.e. ``ssmLT.xml`` for most GEM Hazard Mosaic model repositories. -* ``branch`` - This specifies what branch is to be evaluated. Because different branches +``job_ini_file`` + Alternative to ``ssm_dir`` + ``ssm_lt_file``: path to an OpenQuake + ``job.ini`` file that specifies the source model. This is useful when + the source model configuration is complex. + +``branch`` + Specifies which logic tree branch to evaluate. Because different branches are often mutually-exclusive, alternative descriptions of earthquake occurrence, evaluating multiple branches at once may greatly increase the - forecasted occurrence rates of earthquakes, and result in highly innacurate - evaluation of the model. + forecasted occurrence rates and result in inaccurate evaluation. -* ``tectonic_region_types`` - This parameter is optional but specifies which Tectonic Region Type(s) - should be evaluated. For example, one may want to evaluate different region - types separately, to better understand how well the model predicts - seismicity in active crust vs. stable regions. The types must correspond to - those in the SSM. + Set to ``"iterate"`` to evaluate each branch independently in a single + Hamlet run. This loads the earthquake catalog once and then evaluates each + branch separately, producing combined results. - This can be specified as a list, or by passing ``null`` to consider all - tectonic region types. For example: +``tectonic_region_types`` + Optional filter specifying which Tectonic Region Type(s) should be + evaluated. The types must correspond to those in the SSM. Pass as a YAML + list or omit/set to ``null`` to include all types. -.. code-block:: yaml + .. code-block:: yaml - tectonic_region_types: - - Active Shallow Crust - - Stable Continental + tectonic_region_types: + - Active Shallow Crust + - Stable Continental +``source_types`` + Optional filter specifying which source types to evaluate. Options include + ``simple_fault``, ``complex_fault``, ``area``, ``point``, ``multipoint``, + and ``MultiFaultSource``. Pass as a YAML list, or ``null`` for all types. -* ``source_types`` - This parameter is optional but specifies which source types are evaluated. - Options include ``simple_fault``, ``complex_fault``, ``area``, ``point``, - and ``multipoint``. As with the ``tectonic_region_types``, the values need - to be passed as a YAML list, or the value of ``null`` should be given. - A single value may be passed as -.. code-block:: yaml +Depth Filtering (Optional) +-------------------------- - source_types: - - point +``min_depth`` + Optional. Minimum source depth in km. Sources shallower than this are + excluded from the evaluation. Default: ``0.0``. Specified at the + ``input`` level (not inside ``ssm``). +``max_depth`` + Optional. Maximum source depth in km. Sources deeper than this are + excluded from the evaluation. Default: no limit. Specified at the + ``input`` level (not inside ``ssm``). + .. code-block:: yaml -Observed earthquake catalog (``seis_catalog``) + input: + min_depth: 0 + max_depth: 40 + + +Observed Earthquake Catalog (``seis_catalog``) ---------------------------------------------- This set of parameters determines how the seismic catalog will be found and parsed so that it can be compared to the source model. ``seis_catalog_file`` - This parameter gives the relative or absolute filepath and filename of the - CSV file that is the earthquake catalog. - -``columns`` - This parameter gives a list of the expected column names with the names of - the columns in the CSV file. The first set of column names are optional, - with very common defaults, and only need to be put into the YAML file - if the names to do not correspond to the defaults. If the earthquake - catalog is made from the ISC-GEM or the GEM toolkits, these columns - do not need to be given in the YAML file: - - * ``x_col`` - This defaults to ``longitude``. If this column in the CSV is not - called ``longitude``, please pass the column name. - - * ``y_col`` - This defaults to ``latitude``. If this is not the column name in the - CSV file, please pass the column name. - - * ``depth`` - This defaults to ``depth``. If this is not the column name in the - CSV file, please pass the column name. - - * ``magnitude`` - This defaults to ``magnitude``. If this is not the column name in the - CSV file, please pass the column name. - - The following columns have no defaults; if they are to be used, they should - be set here: - - * ``source`` - This column specifies the institutional source of the earthquake, not - the geological source. In the GEM catalogs, this is commonly - ``Agency``. - - * ``event_id`` - This is the ID of the earthquake. In the GEM catalogs, this is - commonly ``eventID``. - - And finally, a more complex column, which is actually a set of columns in - many seismic catalogs: - - * ``time`` - This sub-parameter is an ordered list of the columns specifying the time - components that are used to make a single time for the earthquake (this - is done by making a :class:`datetime.datetime` time object; please see - `dateutil `_ for more - information). For the typical GEM catalog, the time sub-parameter will - be specified like this: + Relative or absolute filepath to the CSV earthquake catalog file. -.. code-block:: yaml +Temporal Parameters +~~~~~~~~~~~~~~~~~~~ + +Provide any two of ``start_date``, ``stop_date``, and ``duration``; the third +will be calculated automatically. Alternatively, use a ``completeness_table``. + +``start_date`` + Start of the catalog time window. Can be an integer (interpreted as + January 1 of that year) or a date string (e.g. ``"1976-01-01"``). + +``stop_date`` + End of the catalog time window. Same format as ``start_date``. + +``duration`` + Duration of the catalog in years (float). + +``completeness_table`` + A list of ``[year, magnitude]`` pairs defining the completeness threshold + over time. Each pair means "the catalog is complete above this magnitude + from this year onward." When used, the effective duration varies by + magnitude bin, which enables more accurate evaluation of models across + the magnitude range. + + .. code-block:: yaml + + completeness_table: + - [1960, 5.0] + - [1900, 7.2] + +Column Mappings (``columns``) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - time: - - year - - month - - day - - hour - - minute - - second +Maps expected fields to actual column names in the CSV file. Only specify +columns whose names differ from the defaults. +* ``x_col`` -- Defaults to ``longitude``. +* ``y_col`` -- Defaults to ``latitude``. +* ``depth`` -- Defaults to ``depth``. +* ``magnitude`` -- Defaults to ``magnitude``. +* ``source`` -- No default. The institutional source of the earthquake + (e.g. ``Agency``). +* ``event_id`` -- No default. The earthquake ID column (e.g. ``eventID``). +* ``time`` -- Either a single column name containing a parseable timestamp, + or an ordered list of columns to construct the time: + .. code-block:: yaml + time: + - year + - month + - day + - hour + - minute + - second Bins (``bins``) --------------- -This section describes configuration for the -:class:`~openquake.hme.utils.bins.SpacemagBin` s that are used throughout Hamlet. -All of the parameters are optional, and should only be listed in the YAML file -if the defaults are not suitable. +This section configures the spatial and magnitude binning used throughout +Hamlet. -Sub-parameters: +Default values for many parameters (including ``h3_res``, ``simple_ruptures``, +``subset``, and ``rupture_file`` options) are defined in +``openquake.hme.core.core.cfg_defaults`` and are applied automatically when +not specified in the YAML file. -* ``bin_gis_file`` - Optional path to a GIS file with polygons that will become the bins. The GIS - file will be read with `GeoPandas `_ and one can find - information on the acceptable vector GIS filetypes at that link. +``h3_res`` + The resolution of the `H3 `_ hexagonal + grid used for spatial binning. Values range from 0 (coarsest, ~4,000 km + edge length) to 15 (finest). Default: ``3`` (~69 km edge length). + H3 hexagons are generated automatically based on the spatial extent of + the source model. - It is not recommended to use this option. If it is not given, then polygons - are created using Uber's `h3-py `_ library - automatically (based on the spatial extent of the seismic source model), - which results in much faster testing and is far less of a hassle than - creating your own GIS file. However if the polygons should correspond to - e.g. seismic source zones for some purpose, then this option could be used. +``mfd_bin_min`` + Minimum magnitude for the MFD. Required. -* ``mfd_bin_max`` - Maximum size of the model MFD to be considered. Defaults to 9.0. +``mfd_bin_max`` + Maximum magnitude for the MFD. Required. -* ``mfd_bin_min`` - Minimum size of the model MFD to be considered. Defaults to 6.0. +``mfd_bin_width`` + Width of the magnitude bins. Required. -* ``mfd_bin_width`` - Width of the bins for the MFD. Defaults to 0.2. Narrower bins may have - issues with lower-resolution source models or seismic catalogs. -.. _report: +Rupture File Caching (``rupture_file``) +--------------------------------------- + +Loading ruptures from a source model can be slow for large models. These +optional parameters let you save processed ruptures to disk and reload them +in subsequent runs, if you have not changed the model and are just changing the +test configuration (branches, depth ranges, catalog completeness, etc.). + +.. code-block:: yaml + + rupture_file: + read_rupture_file: false + save_rupture_file: false + rupture_file_path: ./ruptures.hdf5 + +``read_rupture_file`` + If ``true``, read ruptures from the file at ``rupture_file_path`` instead + of processing the SSM. Default: ``false``. + +``save_rupture_file`` + If ``true``, save processed ruptures to ``rupture_file_path``. Default: + ``false``. -Reporting -========= +``rupture_file_path`` + Path to the rupture file. Supported formats: ``.hdf5``, ``.feather``, + ``.csv``. -This optional section describes how reports (listing the results of the -evaluations) will be written. Currently there is only one option, the ``basic`` -HTML report. This report will aggregate all of the tests or evaluations -performed and summarize them, either in a map, plot, or table. -It is suggested to use this option instead of writing plots directly -either through the ``model_mfd`` test configuration, or the ``output`` -configuration below, as it is a nicer summary. However if one has specific -goals, then those other options may be used. +Spatial Subsetting (``subset``) +------------------------------- -To write the basic HTML report, put this section in the YAML: +Optional parameters to restrict the evaluation to a geographic subset of +the model domain. .. code-block:: yaml - report: - basic: - outfile: /path/to/file.html + subset: + file: path/to/subset.geojson + buffer: 0.0 -In the future, more types of reports may be generated, but this is sufficient -for now. +``file`` + Path to a GIS file containing the subset geometry. Default: ``null``. +``buffer`` + Buffer distance around the subset geometry (in the units of the GIS file's + coordinate reference system). Default: ``0.0``. -.. _output: +Flatfile (``flatfile``) +----------------------- + +Path to a ground motion flatfile (CSV). Required for the +``catalog_ground_motion_eval`` evaluation. Specified at the ``input`` level. + +.. code-block:: yaml + + input: + flatfile: path/to/flatfile.csv + + +Note that this requires a flatfile that is constructed with a format that is +used internally at GEM (and currently undocumented, pending publication). Until +the format stabilizes, there will almost certainly be problems. -Outputs -======= -This optional section specifies writing other outputs than reports (plots or GIS -files). This option can give more fine-grained control over the outputs than the -``report`` option, at this time. However the results are less convenient than -writing a report. +Other Input Options +------------------- -Parameters: +``simple_ruptures`` + Boolean. Use simplified rupture representations (hypocenter point sources) for faster processing. Default: ``True``. -``bin_gdf`` - If this parameter is passed, a GeoJSON file will be written that contains - much of the information in the Hamlet evaluation, including test results, - for each spatial bin. The filename to be used is given with a sub-parameter - ``file``: + +.. _report: + +Reporting (``report``) +====================== + +This optional section configures report generation. Currently there is one +option, the ``basic`` HTML report, which aggregates all test and evaluation +results into maps, plots, and tables. .. code-block:: yaml - bin_gdf: - file: /path/to/file.geojson + report: + basic: + outfile: outputs/report.html + + +.. _output: +JSON Output (``json``) +====================== -``plots`` - This is a parameter (that may be soon deprecated) describing how an MFD plot - should be written. It can offer more control than the other options, - but is not not thoroughly documented at this time. +Optional. Write test results as a JSON file. + +.. code-block:: yaml + + json: + outfile: outputs/results.json + + +Minimal Example +=============== + +.. literalinclude:: example_configs/minimal_config.yml + :language: yaml -Putting it all together +Comprehensive Reference ======================= -A simple example is taken from the test suite below: +The complete reference configuration with all options documented is available +at ``doc_src/source/example_configs/reference_config.yml`` in the repository. -.. literalinclude:: ../../tests/model_test_frameworks/gem/integration_tests/gem_sm1/test_sm1_poisson.yml +.. literalinclude:: example_configs/reference_config.yml :language: yaml diff --git a/openquake/hme/model_test_frameworks/gem/gem_test_functions.py b/openquake/hme/model_test_frameworks/gem/gem_test_functions.py index 71f8c93..f7a0511 100644 --- a/openquake/hme/model_test_frameworks/gem/gem_test_functions.py +++ b/openquake/hme/model_test_frameworks/gem/gem_test_functions.py @@ -30,6 +30,15 @@ def get_rupture_gdf_cell_moment(rupture_gdf, t_yrs, rup_groups=None): + """Computes the expected seismic moment per spatial cell and total, given + rupture occurrence rates scaled by duration. + + :param rupture_gdf: GeoDataFrame of ruptures with ``magnitude``, + ``occurrence_rate``, and ``cell_id`` columns. + :param t_yrs: Duration in years to scale occurrence rates. + :param rup_groups: Optional pre-computed groupby on ``cell_id``. + :returns: Tuple of (per-cell moment Series, total moment float). + """ if rup_groups == None: rup_groups = rupture_gdf.groupby("cell_id") @@ -50,6 +59,14 @@ def get_rupture_gdf_cell_moment(rupture_gdf, t_yrs, rup_groups=None): def get_catalog_moment(eq_df, eq_groups=None): + """Computes the total seismic moment per spatial cell and overall from an + earthquake catalog. + + :param eq_df: GeoDataFrame of earthquakes with ``magnitude`` and + ``cell_id`` columns. + :param eq_groups: Optional pre-computed groupby on ``cell_id``. + :returns: Tuple of (per-cell moment dict, total moment float). + """ if eq_groups == None: eq_groups = eq_df.groupby("cell_id") @@ -65,6 +82,23 @@ def get_catalog_moment(eq_df, eq_groups=None): def moment_over_under_eval_fn( rup_df, eq_gdf, cell_groups, t_yrs, min_mag=1.0, max_mag=10.0, n_iters=1000 ): + """Compares observed seismic moment release to stochastic moment release + from the model, per cell and in total. + + Generates ``n_iters`` stochastic catalogs by sampling ruptures, computes + moment release for each, and calculates the fractile of the observed + moment within the stochastic distribution. + + :param rup_df: GeoDataFrame of ruptures. + :param eq_gdf: GeoDataFrame of observed earthquakes. + :param cell_groups: Pre-computed groupby of ruptures on ``cell_id``. + :param t_yrs: Duration in years. + :param min_mag: Minimum magnitude for moment calculation. + :param max_mag: Maximum magnitude for moment calculation. + :param n_iters: Number of stochastic catalogs to generate. + :returns: Dict with ``test_data`` containing per-cell and total moment + comparisons and fractiles. + """ cell_ids = sorted(rup_df.cell_id.unique()) cell_model_moments, total_model_moment = get_rupture_gdf_cell_moment( @@ -141,7 +175,20 @@ def model_mfd_eval_fn( annualize=False, stop_date=None, ): - + """Computes and compares model and observed magnitude-frequency + distributions. + + :param rup_gdf: GeoDataFrame of ruptures. + :param eq_gdf: GeoDataFrame of observed earthquakes. + :param mag_bins: Dict of magnitude bin centers to (min, max) tuples. + :param t_yrs: Duration in years. + :param completeness_table: Optional completeness table as list of + [year, magnitude] pairs. + :param annualize: If True, annualize rates. + :param stop_date: End date of the catalog. + :returns: Dict with ``test_data`` containing a DataFrame of model and + observed MFDs (incremental and cumulative). + """ if annualize: t_yrs_model = 1.0 completeness_table_model = None @@ -186,6 +233,11 @@ def model_mfd_eval_fn( def get_moment_from_mfd(mfd: dict) -> float: + """Calculates total seismic moment from an MFD dictionary. + + :param mfd: Dict mapping magnitude bin centers to rates. + :returns: Total seismic moment (N*m). + """ if isinstance(mfd, dict): return _get_moment_from_mfd_dict(mfd) else: @@ -201,6 +253,14 @@ def _get_moment_from_mfd_dict(mfd: dict) -> float: def mag_diff_likelihood(eq_mag, rup_mags, mag_window=1.0): + """Calculates a linear likelihood based on the magnitude difference + between an earthquake and candidate ruptures. + + :param eq_mag: Observed earthquake magnitude. + :param rup_mags: Array of rupture magnitudes. + :param mag_window: Total width of the magnitude window for matching. + :returns: Array of likelihoods in [0, 1], where 1 means exact match. + """ likes = 1 - np.abs(eq_mag - rup_mags) / (mag_window / 2.0) if np.isscalar(likes): if likes < 0.0: @@ -212,6 +272,12 @@ def mag_diff_likelihood(eq_mag, rup_mags, mag_window=1.0): def get_distances(eq, rup_gdf): + """Calculates 3D distances between an earthquake and a set of ruptures. + + :param eq: Earthquake row with ``longitude``, ``latitude``, ``depth``. + :param rup_gdf: GeoDataFrame of ruptures with the same columns. + :returns: Array of distances in km. + """ # this assumes we want 3d distance instead of separate treatment # of h, v dists dists = distance( @@ -226,6 +292,13 @@ def get_distances(eq, rup_gdf): def get_rups_in_mag_range(eq, rup_df, mag_window=1.0): + """Filters ruptures to those within a magnitude window of the earthquake. + + :param eq: Earthquake row with ``magnitude``. + :param rup_df: DataFrame of ruptures with ``magnitude`` column. + :param mag_window: Total width of the magnitude window. + :returns: Filtered DataFrame of ruptures within the window. + """ rdf_lo = rup_df.loc[ rup_df.magnitude.values <= (eq.magnitude + mag_window / 2.0) ] @@ -237,6 +310,12 @@ def get_rups_in_mag_range(eq, rup_df, mag_window=1.0): def get_nearby_rups(eq, rup_df): + """Finds ruptures in the earthquake's H3 cell and its immediate neighbors. + + :param eq: Earthquake row with ``cell_id``. + :param rup_df: DataFrame of ruptures with ``cell_id`` column. + :returns: Filtered DataFrame of nearby ruptures. + """ # first find adjacent cells to pare down search space closest_cells = h3.grid_disk(eq.cell_id, 1) @@ -261,6 +340,38 @@ def get_matching_rups( rake_rel_weight=0.25, mag_rel_weight=1.0, ): + """Finds and ranks modeled ruptures that match an observed earthquake. + + Matching is done in two phases: selection (nearby ruptures within a + magnitude window) and ranking (weighted geometric mean of distance, + magnitude, attitude, and rake likelihoods). If focal mechanism data is + available (single or double-couple), attitude and rake similarity are + included; otherwise, default likelihoods are used. + + :param eq: Earthquake row with location, magnitude, and optional focal + mechanism columns (``strike``, ``dip``, ``rake`` or + ``strike1``/``strike2`` etc.). + :param rup_gdf: GeoDataFrame of candidate ruptures. + :param distance_lambda: Distance decay parameter (scaled by magnitude + if ``dist_by_mag`` is True). + :param dist_by_mag: Scale distance decay by earthquake magnitude. + :param mag_window: Total width of the magnitude window for candidates. + :param group_return_threshold: Fraction of max likelihood below which + matches are discarded. + :param min_likelihood: Absolute minimum likelihood for a match. + :param no_attitude_default_like: Default attitude likelihood when no + focal mechanism is available. + :param no_rake_default_like: Default rake likelihood when no focal + mechanism is available. + :param use_occurrence_rate: Include occurrence rate in the ranking. + :param return_one: ``False`` to return all matches, ``"best"`` for the + top match, ``"sample"`` to sample weighted by likelihood. + :param attitude_rel_weight: Relative weight for attitude similarity. + :param rake_rel_weight: Relative weight for rake similarity. + :param mag_rel_weight: Relative weight for magnitude similarity. + :returns: DataFrame of matching ruptures (or Series if ``return_one``), + or ``None`` if no matches found. + """ # selection phase rups = get_nearby_rups(eq, rup_df=rup_gdf) rups = get_rups_in_mag_range(eq, rup_df=rups, mag_window=mag_window) @@ -485,6 +596,22 @@ def match_eqs_to_rups( return_one="best", parallel=False, ): + """Matches all earthquakes in a catalog to their best-matching modeled + ruptures using :func:`get_matching_rups`. + + :param eq_gdf: GeoDataFrame of observed earthquakes. + :param rup_gdf: GeoDataFrame of modeled ruptures. + :param distance_lambda: Distance decay parameter. + :param dist_by_mag: Scale distance decay by earthquake magnitude. + :param mag_window: Magnitude window for candidate ruptures. + :param group_return_threshold: Fraction of max likelihood threshold. + :param no_attitude_default_like: Default attitude likelihood. + :param no_rake_default_like: Default rake likelihood. + :param use_occurrence_rate: Include occurrence rate in ranking. + :param return_one: ``"best"``, ``"sample"``, or ``False``. + :param parallel: Use multiprocessing (currently disabled). + :returns: List of match results (DataFrames or None) per earthquake. + """ match_rup_args = ( ( eq, @@ -531,6 +658,14 @@ def rupture_matching_eval_fn( return_one="best", parallel=False, ): + """Runs the rupture matching evaluation, matching all observed earthquakes + to modeled ruptures and collecting matched/unmatched results. + + :param rup_gdf: GeoDataFrame of modeled ruptures. + :param eq_gdf: GeoDataFrame of observed earthquakes. + :returns: Dict with ``matched_rups`` DataFrame and ``unmatched_eqs`` + DataFrame. + """ match_results = match_eqs_to_rups( eq_gdf, rup_gdf, @@ -593,10 +728,22 @@ def rupture_matching_eval_fn( def get_flatfile_records_for_eq(event_id, gm_df: pd.DataFrame) -> pd.DataFrame: + """Returns all flatfile records for a given earthquake event ID.""" return gm_df.loc[gm_df.event_id == event_id] def predict_gms_for_eq(eq, gm_df, gsim_lt, test_config, imts=(PGA(),)): + """Predicts ground motions at recording sites for a given earthquake using + the ground motion models from the logic tree, and compares with observed + values from the flatfile. + + :param eq: Earthquake (or rupture object). + :param gm_df: Ground motion flatfile DataFrame. + :param gsim_lt: Ground motion model logic tree. + :param test_config: Test configuration dict with ``gmf_method``. + :param imts: Tuple of intensity measure types (default: PGA). + :returns: DataFrame with observed and predicted ground motions per site. + """ site_records = get_flatfile_records_for_eq(eq.event_id, gm_df) sitecol = make_sitecol( site_records["st_longitude"].values, @@ -653,12 +800,25 @@ def predict_gms_for_eq(eq, gm_df, gsim_lt, test_config, imts=(PGA(),)): def get_closest_rupture(eq, rupture_df): + """Returns the rupture closest to the given earthquake in 3D distance.""" dists = get_distances(eq, rupture_df) return rupture_df.iloc[dists.argmin()] def catalog_ground_motion_eval_fn(test_config, input_data): - + """Compares observed ground motions from a flatfile with model predictions. + + For each earthquake, either matches it to a modeled rupture or constructs + a rupture from the flatfile data, then computes predicted ground motion + fields using the ground motion models from the source model logic tree. + + :param test_config: Test configuration dict with matching parameters and + ``gmf_method``. + :param input_data: Dict with ``rupture_gdf``, ``eq_gm_df``, ``gm_df``, + and ``gsim_lt``. + :returns: Dict keyed by tectonic region type, with DataFrames of observed + vs. predicted ground motions per earthquake. + """ # defining this here for now, will go in config later imts = (PGA(),) diff --git a/openquake/hme/model_test_frameworks/gem/gem_tests.py b/openquake/hme/model_test_frameworks/gem/gem_tests.py index 506a8cc..94f9cf8 100644 --- a/openquake/hme/model_test_frameworks/gem/gem_tests.py +++ b/openquake/hme/model_test_frameworks/gem/gem_tests.py @@ -113,7 +113,10 @@ def S_test( cfg: dict, input_data: dict, ) -> dict: - """""" + """GEM S-Test: evaluates the spatial consistency of the model by comparing + per-cell likelihoods of the observed catalog against stochastic catalogs. + Highlights cells where the model over- or under-predicts seismicity. + """ logging.info("Running GEM S-Test") mag_bins = get_mag_bins_from_cfg(cfg) @@ -164,7 +167,9 @@ def L_test( cfg: dict, input_data: dict, ) -> dict: - """""" + """GEM L-Test: joint likelihood test combining spatial and magnitude + information to evaluate overall model consistency. + """ logging.info("Running GEM L-Test") mag_bins = get_mag_bins_from_cfg(cfg) @@ -207,6 +212,9 @@ def L_test( def N_test(cfg: dict, input_data: dict) -> dict: + """GEM N-Test: compares the total observed earthquake count to the number + predicted by the model using a Poisson confidence interval. + """ logging.info("Running N-Test") test_config = cfg["config"]["model_framework"]["gem"]["N_test"] @@ -256,6 +264,9 @@ def N_test(cfg: dict, input_data: dict) -> dict: def max_mag_check(cfg: dict, input_data: dict): + """Checks whether the model can produce earthquakes as large as the largest + observed earthquake in each spatial cell. + """ logging.info("Checking Maximum Magnitudes") max_bin_check_results = max_check(cfg, input_data, framework="gem") @@ -281,6 +292,9 @@ def max_mag_check(cfg: dict, input_data: dict): def model_mfd_eval(cfg, input_data): + """Compares the total model MFD to the observed MFD from the earthquake + catalog, producing incremental and cumulative MFD data for reporting. + """ logging.info("Running GEM Model MFD Eval") mag_bins = get_mag_bins_from_cfg(cfg) completeness_table = cfg["input"]["seis_catalog"].get("completeness_table") @@ -315,6 +329,9 @@ def model_mfd_eval(cfg, input_data): def moment_over_under_eval(cfg, input_data): + """Compares observed vs. stochastic seismic moment release per cell and + in total, to highlight areas of over- or under-prediction. + """ logging.info("Running GEM Moment Over-Under Eval") test_config = cfg["config"]["model_framework"]["gem"]["moment_over_under"] @@ -373,6 +390,9 @@ def moment_over_under_eval(cfg, input_data): def rupture_matching_eval(cfg, input_data): + """Matches observed earthquakes to modeled ruptures based on proximity, + magnitude, and (optionally) focal mechanism similarity. + """ logging.info("Running GEM Rupture Matching Eval") test_config = cfg["config"]["model_framework"]["gem"][ @@ -430,11 +450,15 @@ def rupture_matching_eval(cfg, input_data): def mfd_likelihood_test(cfg, input_data): + """.. deprecated:: Use ``M_test``, ``S_test``, and ``L_test`` instead.""" logging.warning("GEM Likelihood test deprecated") return def cumulative_occurrence_eval(cfg, input_data): + """Evaluates cumulative earthquake occurrence over time for each magnitude + bin, comparing the observed temporal pattern to the model rate. + """ logging.info("Running GEM Cumultive Earthquake Occurrence Eval") eqs = input_data["eq_gdf"] @@ -480,7 +504,9 @@ def cumulative_occurrence_eval(cfg, input_data): def catalog_ground_motion_eval(cfg, input_data): - + """Compares observed ground motions from a flatfile with model-predicted + ground motions. Requires ``input.flatfile`` in the configuration. + """ logging.info("Running GEM catalog ground motion evaluation") test_config = cfg["config"]["model_framework"]["gem"][ diff --git a/openquake/hme/model_test_frameworks/relm/relm_test_functions.py b/openquake/hme/model_test_frameworks/relm/relm_test_functions.py index 54305d0..dbea0e1 100644 --- a/openquake/hme/model_test_frameworks/relm/relm_test_functions.py +++ b/openquake/hme/model_test_frameworks/relm/relm_test_functions.py @@ -45,6 +45,24 @@ def l_test_function( critical_frac: float = 0.25, not_modeled_likelihood: float = 0.0, ): + """Joint likelihood test (L-test). Computes the total log-likelihood of + the observed catalog across all spatial cells and magnitude bins, and + compares it to the distribution of log-likelihoods from stochastic + catalogs. + + :param rup_gdf: GeoDataFrame of modeled ruptures. + :param eq_gdf: GeoDataFrame of observed earthquakes. + :param cell_groups: Groupby of ruptures on ``cell_id``. + :param eq_groups: Groupby of earthquakes on ``cell_id``. + :param t_yrs: Investigation time in years. + :param n_iters: Number of Monte Carlo iterations. + :param mag_bins: Magnitude bin dictionary. + :param completeness_table: Optional completeness table. + :param stop_date: End date of the catalog. + :param critical_frac: Fractile threshold for test failure. + :param not_modeled_likelihood: Likelihood for unmodeled cells/bins. + :returns: Dict with test result, fractile, and test data. + """ cell_like_cfg = { "investigation_time": t_yrs, "completeness_table": completeness_table, @@ -110,6 +128,24 @@ def m_test_function( critical_frac: float = 0.25, normalize_n_eqs: Optional[bool] = True, ): + """Magnitude consistency test (M-test). Evaluates the consistency of the + model MFD vs. the observed MFD by comparing the geometric mean + log-likelihood of the observed catalog against stochastic catalogs + generated from the model. + + :param rup_gdf: GeoDataFrame of modeled ruptures. + :param eq_gdf: GeoDataFrame of observed earthquakes. + :param mag_bins: Magnitude bin dictionary. + :param t_yrs: Investigation time in years. + :param n_iters: Number of Monte Carlo iterations. + :param completeness_table: Optional completeness table. + :param stop_date: End date of the catalog. + :param not_modeled_likelihood: Likelihood for unmodeled magnitude bins. + :param critical_frac: Fractile threshold for test failure. + :param normalize_n_eqs: If True, normalize stochastic catalogs to match + the observed earthquake count. + :returns: Dict with test result, fractile, and test data. + """ # normalized to duration !! mod_mfd = get_model_mfd( @@ -227,6 +263,28 @@ def s_test_function( not_modeled_likelihood: float = 0.0, parallel: bool = False, ): + """Spatial consistency test (S-test). Evaluates the spatial distribution of + the model by comparing per-cell log-likelihoods of the observed catalog + against stochastic catalogs. Supports multiple likelihood functions. + + :param rup_gdf: GeoDataFrame of modeled ruptures. + :param eq_gdf: GeoDataFrame of observed earthquakes. + :param cell_groups: Groupby of ruptures on ``cell_id``. + :param eq_groups: Groupby of earthquakes on ``cell_id``. + :param t_yrs: Investigation time in years. + :param n_iters: Number of Monte Carlo iterations. + :param likelihood_fn: Likelihood function name (``"mfd"``, + ``"n_eqs"``, or ``"conf_interval_poisson"``). + :param mag_bins: Magnitude bin dictionary. + :param normalize_n_eqs: Normalize stochastic catalogs to observed count. + :param completeness_table: Optional completeness table. + :param stop_date: End date of the catalog. + :param critical_frac: Fractile threshold for test failure. + :param not_modeled_likelihood: Likelihood for unmodeled cells. + :param parallel: Use multiprocessing for cell-level computation. + :returns: Dict with test result, fractile, per-cell fractions, and + test data. + """ if normalize_n_eqs: obs_mfd = get_obs_mfd( eq_gdf, @@ -357,6 +415,11 @@ def s_test_function( def s_test_cells( cell_groups, rup_gdf, eq_groups, eq_gdf, test_cfg, parallel: bool = False ): + """Runs the S-test likelihood computation for each spatial cell, + optionally in parallel. + + :returns: Dict mapping cell IDs to per-cell likelihood results. + """ s_test_cell_results = {} cell_ids = sorted(cell_groups.groups.keys()) @@ -389,7 +452,15 @@ def _s_test_cell_args(cell_args): def s_test_cell(rup_gdf, eq_gdf, test_cfg): - """""" + """Computes the S-test likelihood for a single spatial cell by comparing + the observed MFD log-likelihood to stochastic MFD log-likelihoods. + + :param rup_gdf: Ruptures within this cell. + :param eq_gdf: Earthquakes within this cell. + :param test_cfg: Test configuration dict. + :returns: Dict with observed and stochastic log-likelihoods, bad bins, + and unmatched earthquakes. + """ cell_id = rup_gdf.cell_id.values[0] t_yrs = test_cfg["investigation_time"] completeness_table = test_cfg.get("completeness_table") @@ -613,6 +684,18 @@ def conf_interval_poisson( return_data: bool = False, **kwargs, ) -> float: + """Evaluates whether the observed total event count falls within a Poisson + confidence interval of the modeled rate. Also generates stochastic samples + to assess the fraction of samples that pass. + + :param rate_mfd: Dict of modeled MFD rates by magnitude bin. + :param empirical_mfd: Dict of observed event counts by magnitude bin. + :param N_norm: Normalization factor for model rates. + :param N_iters: Number of stochastic samples. + :param conf_interval: Confidence interval (e.g. 0.95). + :returns: Dict with observed log-likelihood, stochastic likelihoods, + and cell pass/fail. + """ if binned_events is not None: if empirical_mfd is None: num_obs_events = { @@ -665,6 +748,17 @@ def conf_interval_poisson( def n_test_function(rup_gdf, eq_gdf, test_config: dict): + """N-test: compares the total observed earthquake count to the model + prediction. Supports Poisson, cumulative Poisson, and negative binomial + probability models. + + :param rup_gdf: GeoDataFrame of modeled ruptures. + :param eq_gdf: GeoDataFrame of observed earthquakes. + :param test_config: Test configuration dict with ``prob_model``, + ``conf_interval``, and other parameters. + :returns: Dict with test result, confidence interval, observed and + predicted counts. + """ prospective = test_config.get("prospective", False) conf_interval = test_config.get("conf_interval", 0.95) @@ -739,12 +833,21 @@ def n_test_function(rup_gdf, eq_gdf, test_config: dict): def get_poisson_counts_from_mfd(mfd: dict): + """Samples random Poisson counts from each magnitude bin rate.""" return {mag: np.random.poisson(rate) for mag, rate in mfd.items()} def N_test_empirical( num_obs_events: int, num_pred_events: Sequence[int], conf_interval: float ) -> dict: + """N-test using empirical (cumulative Poisson) distribution from sampled + MFD counts. + + :param num_obs_events: Observed earthquake count. + :param num_pred_events: Sequence of stochastic event counts. + :param conf_interval: Confidence interval fraction. + :returns: Dict with test result, confidence interval, and counts. + """ rupture_rate = np.mean(num_pred_events) conf_half_interval = conf_interval / 2 conf_min = np.percentile(num_pred_events, 100 * (0.5 - conf_half_interval)) @@ -770,6 +873,13 @@ def N_test_empirical( def N_test_poisson( num_obs_events: int, rupture_rate: float, conf_interval: float ) -> dict: + """N-test using a Poisson distribution. + + :param num_obs_events: Observed earthquake count. + :param rupture_rate: Total predicted rate (events per investigation time). + :param conf_interval: Confidence interval fraction. + :returns: Dict with test result, confidence interval, and counts. + """ conf_min, conf_max = poisson(rupture_rate).interval(conf_interval) test_pass = conf_min <= num_obs_events <= conf_max @@ -796,6 +906,17 @@ def N_test_neg_binom( r_dispersion: float, conf_interval: float, ) -> dict: + """N-test using a negative binomial distribution (for temporally + overdispersed earthquake production). Falls back to Poisson if + underdispersed. + + :param num_obs_events: Observed earthquake count. + :param rupture_rate: Predicted rate. + :param prob_success: Negative binomial success probability. + :param r_dispersion: Negative binomial dispersion parameter. + :param conf_interval: Confidence interval fraction. + :returns: Dict with test result, confidence interval, and counts. + """ if r_dispersion < 1: logging.warn( "Earthquake production temporally underdispersed, \n" diff --git a/openquake/hme/model_test_frameworks/relm/relm_tests.py b/openquake/hme/model_test_frameworks/relm/relm_tests.py index e9d8134..f281a15 100644 --- a/openquake/hme/model_test_frameworks/relm/relm_tests.py +++ b/openquake/hme/model_test_frameworks/relm/relm_tests.py @@ -14,6 +14,9 @@ def M_test(cfg, input_data): + """RELM/CSEP M-Test: evaluates the consistency of the model MFD vs. + the observed MFD. Uses ``not_modeled_likelihood=0.0`` (strict). + """ logging.info("Running CSEP/RELM M-Test") mag_bins = get_mag_bins_from_cfg(cfg) @@ -54,7 +57,9 @@ def S_test( cfg: dict, input_data: dict, ) -> dict: - """""" + """RELM/CSEP S-Test: evaluates the spatial consistency of the model. + Uses ``not_modeled_likelihood=0.0`` (strict). + """ logging.info("Running CSEP/RELM S-Test") mag_bins = get_mag_bins_from_cfg(cfg) @@ -106,7 +111,9 @@ def L_test( cfg: dict, input_data: dict, ) -> dict: - """""" + """RELM/CSEP L-Test: joint likelihood test combining spatial and magnitude + information. Uses ``not_modeled_likelihood=0.0`` (strict). + """ logging.info("Running CSEP/RELM L-Test") mag_bins = get_mag_bins_from_cfg(cfg) @@ -149,6 +156,9 @@ def L_test( def N_test(cfg: dict, input_data: dict) -> dict: + """RELM/CSEP N-Test: compares the total observed earthquake count to the + model prediction. + """ logging.info("Running N-Test") logging.info( diff --git a/openquake/hme/model_test_frameworks/sanity/sanity_checks.py b/openquake/hme/model_test_frameworks/sanity/sanity_checks.py index cc082e1..747d220 100644 --- a/openquake/hme/model_test_frameworks/sanity/sanity_checks.py +++ b/openquake/hme/model_test_frameworks/sanity/sanity_checks.py @@ -13,6 +13,15 @@ def min_max_check(): def max_check(cfg, input_data, framework="sanity"): + """Checks whether the model maximum magnitude exceeds the observed maximum + magnitude in each spatial cell. + + :param cfg: Configuration dict. + :param input_data: Dict with rupture and earthquake data. + :param framework: Framework name for config lookup (``"sanity"`` or + ``"gem"``). + :returns: Dict mapping cell IDs to True (pass) or False (fail). + """ bin_width = cfg["input"]["bins"]["mfd_bin_width"] test_cfg = cfg["config"]["model_framework"][framework] diff --git a/setup.py b/setup.py index 4a516b9..be767ae 100644 --- a/setup.py +++ b/setup.py @@ -14,10 +14,10 @@ # Package meta-data. NAME = "openquake.hme" DESCRIPTION = "Hamlet: Hazard model evaluation and testing" -URL = "https://gitlab.com/cossatot/hamlet" -EMAIL = "richard.styron@globbalquakemodel.org" +URL = "https://github.com/GEMScienceTools/hamlet" +EMAIL = "richard.styron@globalquakemodel.org" AUTHOR = "Richard Styron, GEM Foundation" -REQUIRES_PYTHON = ">=3.6.0" +REQUIRES_PYTHON = ">=3.11.0" VERSION = None # What packages are required for this module to be executed? @@ -121,7 +121,9 @@ def run(self): "License :: OSI Approved :: MIT License", "Programming Language :: Python", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", "Programming Language :: Python :: Implementation :: CPython", "Programming Language :: Python :: Implementation :: PyPy", ],