diff --git a/.circleci/config.yml b/.circleci/config.yml index 5617d1b77d39..3ca727a12dc6 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,911 +1,16 @@ version: 2.1 -######################################################################################################################## -# EXECUTORS # -######################################################################################################################## - -executors: - # CircleCI base Node + Headless browsers + Clojure CLI - big one - - builder: - working_directory: /home/circleci/metabase/metabase/ - docker: - - image: metabase/ci:java-11-clj-1.11.0.1100.04-2022-build - - # Java 11 tests also test Metabase with the at-rest encryption enabled. See - # https://metabase.com/docs/latest/operations-guide/encrypting-database-details-at-rest.html for an explanation of - # what this means. - java-11: - working_directory: /home/circleci/metabase/metabase/ - docker: - - image: metabase/ci:java-11-clj-1.11.0.1100.04-2022-build - environment: - MB_ENCRYPTION_SECRET_KEY: Orw0AAyzkO/kPTLJRxiyKoBHXa/d6ZcO+p+gpZO/wSQ= - - java-17: - working_directory: /home/circleci/metabase/metabase/ - docker: - - image: metabase/ci:java-17-clj-1.11.0.1100.04-2022-build - - postgres-9-6: - working_directory: /home/circleci/metabase/metabase/ - docker: - - image: metabase/ci:java-11-clj-1.11.0.1100.04-2022-build - environment: - MB_DB_TYPE: postgres - MB_DB_PORT: 5432 - MB_DB_HOST: localhost - MB_DB_DBNAME: circle_test - MB_DB_USER: circle_test - MB_POSTGRESQL_TEST_USER: circle_test - - image: circleci/postgres:9.6-alpine - environment: - POSTGRES_USER: circle_test - POSTGRES_DB: circle_test - - postgres-latest: - working_directory: /home/circleci/metabase/metabase/ - docker: - - image: metabase/ci:java-11-clj-1.11.0.1100.04-2022-build - environment: - MB_DB_TYPE: postgres - MB_DB_PORT: 5432 - MB_DB_HOST: localhost - MB_DB_DBNAME: metabase_test - MB_DB_USER: metabase_test - MB_POSTGRESQL_TEST_USER: metabase_test - - image: circleci/postgres:latest - environment: - POSTGRES_USER: metabase_test - POSTGRES_DB: metabase_test - POSTGRES_HOST_AUTH_METHOD: trust - - mysql-5-7: - working_directory: /home/circleci/metabase/metabase/ - docker: - - image: metabase/ci:java-11-clj-1.11.0.1100.04-2022-build - environment: - MB_DB_TYPE: mysql - MB_DB_HOST: localhost - MB_DB_PORT: 3306 - MB_DB_DBNAME: circle_test - MB_DB_USER: root - MB_MYSQL_TEST_USER: root - - image: circleci/mysql:5.7.23 - - mysql-latest: - working_directory: /home/circleci/metabase/metabase/ - docker: - - image: metabase/ci:java-11-clj-1.11.0.1100.04-2022-build - environment: - MB_DB_TYPE: mysql - MB_DB_HOST: localhost - MB_DB_PORT: 3306 - MB_DB_DBNAME: circle_test - MB_DB_USER: root - MB_MYSQL_TEST_USER: root - - image: circleci/mysql:latest - - mariadb-10-2: - working_directory: /home/circleci/metabase/metabase/ - docker: - - image: metabase/ci:java-11-clj-1.11.0.1100.04-2022-build - environment: - MB_DB_TYPE: mysql - MB_DB_HOST: localhost - MB_DB_PORT: 3306 - MB_DB_DBNAME: circle_test - MB_DB_USER: root - MB_MYSQL_TEST_USER: root - - image: circleci/mariadb:10.2.23 - - mariadb-latest: - working_directory: /home/circleci/metabase/metabase/ - docker: - - image: metabase/ci:java-11-clj-1.11.0.1100.04-2022-build - environment: - MB_DB_TYPE: mysql - MB_DB_HOST: localhost - MB_DB_PORT: 3306 - MB_DB_DBNAME: circle_test - MB_DB_USER: root - MB_MYSQL_TEST_USER: root - - image: circleci/mariadb:latest - environment: - # MYSQL_DATABASE: metabase_test - # MYSQL_USER: root - # MYSQL_ALLOW_EMPTY_PASSWORD: yes - - mongo-4-0: - working_directory: /home/circleci/metabase/metabase/ - docker: - - image: metabase/ci:java-11-clj-1.11.0.1100.04-2022-build - - image: metabase/qa-databases:mongo-sample-4.0 - - mongo-4-0-ssl: - working_directory: /home/circleci/metabase/metabase/ - docker: - - image: metabase/ci:java-11-clj-1.11.0.1100.04-2022-build - environment: - MB_TEST_MONGO_REQUIRES_SSL: true - - image: metabase/qa-databases:mongo-sample-4.0 - command: mongod --dbpath /data/db2/ --sslMode requireSSL --sslPEMKeyFile /etc/mongo/metamongo.pem --sslCAFile /etc/mongo/metaca.crt - - mongo-5-0: - working_directory: /home/circleci/metabase/metabase/ - docker: - - image: metabase/ci:java-11-clj-1.11.0.1100.04-2022-build - - image: metabase/qa-databases:mongo-sample-5.0 - - mongo-5-0-ssl: - working_directory: /home/circleci/metabase/metabase/ - docker: - - image: metabase/ci:java-11-clj-1.11.0.1100.04-2022-build - environment: - MB_TEST_MONGO_REQUIRES_SSL: true - - image: metabase/qa-databases:mongo-sample-5.0 - command: mongod --dbpath /data/db2/ --tlsMode requireTLS --tlsCertificateKeyFile /etc/mongo/metamongo.pem --tlsCAFile /etc/mongo/metaca.crt - - mongo-latest: - working_directory: /home/circleci/metabase/metabase/ - docker: - - image: metabase/ci:java-11-clj-1.11.0.1100.04-2022-build - - image: circleci/mongo:latest - environment: - MONGO_INITDB_ROOT_USERNAME: metabase - MONGO_INITDB_ROOT_PASSWORD: metasample123 - - - presto-186: - working_directory: /home/circleci/metabase/metabase/ - docker: - - image: metabase/ci:java-11-clj-1.11.0.1100.04-2022-build - - image: metabase/presto-mb-ci:0.186 - environment: - JAVA_TOOL_OPTIONS: "-Xmx2g" - # Run instance with 8GB or RAM instead of the default 4GB for medium instances. The Presto Docker image runs - # OOM sometimes with the default medium size. - resource_class: large - - presto-jdbc-env: - working_directory: /home/circleci/metabase/metabase/ - docker: - - image: metabase/ci:java-11-clj-1.11.0.1100.04-2022-build - - image: metabase/presto-mb-ci:latest # version 0.254 - environment: - JAVA_TOOL_OPTIONS: "-Xmx2g" - MB_PRESTO_JDBC_TEST_CATALOG: test_data - MB_PRESTO_JDBC_TEST_HOST: localhost - MB_PRESTO_JDBC_TEST_PORT: 8443 - MB_PRESTO_JDBC_TEST_SSL: true - MB_PRESTO_JDBC_TEST_USER: metabase - MB_PRESTO_JDBC_TEST_PASSWORD: metabase - MB_ENABLE_PRESTO_JDBC_DRIVER: true - MB_PRESTO_JDBC_TEST_ADDITIONAL_OPTIONS: > - SSLTrustStorePath=/tmp/cacerts-with-presto-ssl.jks&SSLTrustStorePassword=changeit - # (see above) - resource_class: large - - sparksql: - working_directory: /home/circleci/metabase/metabase/ - docker: - - image: metabase/ci:java-11-clj-1.11.0.1100.04-2022-build - - image: metabase/spark:3.2.1 - resource_class: large - - vertica: - working_directory: /home/circleci/metabase/metabase/ - docker: - - image: metabase/ci:java-11-clj-1.11.0.1100.04-2022-build - - image: sumitchawla/vertica - - sqlserver: - working_directory: /home/circleci/metabase/metabase/ - docker: - - image: metabase/ci:java-11-clj-1.11.0.1100.04-2022-build - environment: - MB_SQLSERVER_TEST_HOST: localhost - MB_SQLSERVER_TEST_PASSWORD: 'P@ssw0rd' - MB_SQLSERVER_TEST_USER: SA - - image: mcr.microsoft.com/mssql/server:2017-latest - environment: - ACCEPT_EULA: Y - SA_PASSWORD: 'P@ssw0rd' - MSSQL_MEMORY_LIMIT_MB: 1024 - - druid: - working_directory: /home/circleci/metabase/metabase/ - docker: - - image: metabase/ci:java-11-clj-1.11.0.1100.04-2022-build - - image: metabase/druid:0.20.2 - environment: - CLUSTER_SIZE: nano-quickstart - # Run Docker images with 8GB or RAM instead of the default 4GB for medium instances. The Druid Docker image runs - # OOM all the time with the default medium size. - resource_class: large - -######################################################################################################################## -# MAP FRAGMENTS AND CACHE KEYS # -######################################################################################################################## - -# `default_parameters` isn't a key that CircleCI uses, but this form lets us reuse parameter definitions -default_parameters: &Params - edition: - type: enum - enum: ["oss", "ee"] - default: "oss" - -# .BACKEND-CHECKSUMS and .MODULE-CHECKSUMS are created during the checkout step; see that step -# for exact details as to what they contain. -# -# To support cache busting, we create a file named .CACHE-PREFIX in the checkout step and use its checksum as the -# prefix for every cache key. If the commit message DOES NOT include [ci nocache], we create an empty file; the -# checksum will always be the same for this file. If the commit message DOES include [ci nocache], we'll write the -# unique ID of the current pipeline to .CACHE-PREFIX which will effectively bust our caches whenever it's used. - -### Deps Keys ### - -# Why don't we use fallback keys for backend deps? We used to, but it allowed the cache to grow -# uncontrollably since old deps would continue to accumulate. Restoring big caches is really slow in Circle. It's -# actually faster to recreate the deps cache from scratch whenever we need to which keeps the size down. -cache-key-backend-deps: &CacheKeyBackendDeps - # TODO -- this should actually include the Java source files and the Spark SQL AOT source files as well since we now - # compile those as part of this step. FIXME - key: v5-{{ checksum ".CACHE-PREFIX" }}-be-deps-{{ checksum "deps.edn" }}-{{ checksum ".SCRIPTS-DEPS-CHECKSUMS" }} - -# Key used for implementation of run-on-change -- this is the cache key that contains the .SUCCESS dummy file -# By default the key ALWAYS includes the name of the test job itself ($CIRCLE_JOB) so you don't need to add that yourself. -cache-key-run-on-change: &CacheKeyRunOnChange - key: v5-{{ checksum ".CACHE-PREFIX" }}-run-on-change-{{ .Environment.CIRCLE_JOB }}-<< parameters.checksum >> - -######################################################################################################################## -# COMMANDS # -######################################################################################################################## - -commands: - attach-workspace: - steps: - - attach_workspace: - at: /home/circleci/ - - # For the restore-deps-cache commands below, only restore the cache if there's an exact match. This means whatever - # is in the cache will be exactly what's used and the cache won't keep growing uncontrollably going forward. - - restore-be-deps-cache: - steps: - - restore_cache: - name: Restore cached backend dependencies - <<: *CacheKeyBackendDeps - - # run-on-change lets you only run steps if changes have happened to relevant files since the last time it was run - # successfully. Uses a cache key to record successful runs -- cache key should be unique for job and relevant source - # files -- use a checksum! It works like this: - # - # 1. Calculate a cache key using a checksum of relevant files for the step in question, e.g. a backend linter step - # might use a checksum of all .clj files. - # - # 2. When the step completes successfully, create a dummy file .SUCCESS and cache it with that cache key. - # - # 3. On subsequent runs: - # - # a. Attempt to restore the cache using an exact match for this cache key - # - # b. If we have a cache entry for that key, .SUCCESS will get restored - # - # c. If this command has the skip-job-if-commit-message-includes-ci-quick option enabled, and commit message includes - # [ci quick], create a dummy file .SUCCESS if not already present. Ignored for master/release branches. - # - # d. If commit message includes [ci noskip], delete .SUCCESS so the job will be forced to run. - # - # e. If .SUCCESS is present, we can skip the rest of the job, including potentially slow steps like restoring - # dependency caches or the like. This logs a link to the last successful (not skipped) run of the job - # - # Important! If this step is skipped because no changes have happened, the entire JOB will halt with a success - # status -- no steps that happen AFTER run-on-change will be ran. Keep this in mind! - # - # f. If .SUCCESS is not present, proceed as normal, and create and cache .SUCCESS if the job succeeds - run-on-change: - parameters: - checksum: - type: string - default: "" - steps: - type: steps - # Whether to skip the rest of the job if commit message includes [ci quick] - skip-job-if-commit-message-includes-ci-quick: - type: boolean - default: false - steps: - - restore_cache: - name: Restore dummy file .SUCCESS if it exists for cache key << parameters.checksum >> - <<: *CacheKeyRunOnChange - - when: - condition: << parameters.skip-job-if-commit-message-includes-ci-quick >> - steps: - - run: - name: "Skip tests (create dummy file .SUCCESS) if commit message contains [ci quick] and branch isn't a master/release branch" - command: | - if [[ "$CIRCLE_BRANCH" =~ ^master|release-.+$ ]]; then - echo "branch '$CIRCLE_BRANCH' is a master or release branch: ignoring [ci quick]" - elif [[ `cat .COMMIT` == *"[ci quick]"* ]]; then - echo 'Commit message includes [ci quick]. Creating dummy file .SUCCESS' - touch .SUCCESS - else - echo 'Commit message does not include [ci quick]' - fi - - run: - name: "Force test run (delete dummy file .SUCCESS) if commit message includes [ci noskip]" - command: | - if [[ `cat .COMMIT` == *"[ci noskip]"* ]]; then - echo 'Commit message includes [ci noskip] -- forcing test run (delete .SUCCESS)' - rm -f .SUCCESS - else - echo 'Commit message does not include [ci noskip]' - fi - - run: - name: Skip rest of job if .SUCCESS exists - command: | - if [ -f .SUCCESS ]; then - echo '.SUCCESS is present: skipping rest of job.' - echo "Link to last successful run (if available): $(cat .SUCCESS)" - circleci-agent step halt - fi - - steps: << parameters.steps >> - - run: - name: Create dummy file .SUCCESS - command: | - echo "$CIRCLE_BUILD_URL" > .SUCCESS - - save_cache: - name: Persist dummy file .SUCCESS to cache with key << parameters.checksum >> - <<: *CacheKeyRunOnChange - paths: - - /home/circleci/metabase/metabase/.SUCCESS - - run: - name: Delete dummy file .SUCCESS so subsequent steps don't see it - command: rm /home/circleci/metabase/metabase/.SUCCESS - - # Creates a file that contains checksums for all the files found using the find command with supplied arguments. - # You can use a checksum of the checksum file for cache keys including run-on-change cache keys. - create-checksum-file: - parameters: - filename: - type: string - find-args: - type: string - steps: - - run: - name: Create << parameters.filename >> checksum file - command: | - for file in `find << parameters.find-args >> | sort`; do - echo `md5sum "$file"` >> "<< parameters.filename >>" - done - if [ ! -f "<< parameters.filename >>" ]; then - echo 'Error: no matching files. Did you remember to attach the workspace?' - exit 1 - fi - echo "Created checksums for $(cat << parameters.filename >> | wc -l) files" - - run-clojure-command: - parameters: - before-steps: - type: steps - default: [] - clojure-args: - type: string - after-steps: - type: steps - default: [] - <<: *Params - steps: - - restore-be-deps-cache - - steps: << parameters.before-steps >> - - run: - name: clojure << parameters.clojure-args >>:<< parameters.edition >>:<< parameters.edition >>-dev - command: | - clojure << parameters.clojure-args >>:<< parameters.edition >>:<< parameters.edition >>-dev - no_output_timeout: 15m - - steps: << parameters.after-steps >> - - store_test_results: - path: /home/circleci/metabase/metabase/target/junit - - wait-for-port: - parameters: - port: - type: integer - steps: - - run: - name: Wait for port << parameters.port >> to be ready - command: | - while ! nc -z localhost << parameters.port >>; do sleep 0.1; done - no_output_timeout: 15m - - fetch-jdbc-driver: - parameters: - source: - type: string - dest: - type: string - steps: - - run: - name: Make plugins dir - command: mkdir /home/circleci/metabase/metabase/plugins - - run: - name: Download JDBC driver JAR << parameters.dest >> - command: | - wget --output-document=plugins/<< parameters.dest >> ${<< parameters.source >>} - no_output_timeout: 15m - - run-command: - parameters: - command: - type: string - steps: - - run: - name: Run command - command: << parameters.command >> - +workflows: + empty: + jobs: + - empty jobs: + empty: + docker: + - image: cimg/base:stable -######################################################################################################################## -# CHECKOUT ETC. # -######################################################################################################################## - - checkout: - executor: builder steps: - - checkout - - attach-workspace - # .BACKEND-CHECKSUMS is every Clojure source file as well as dependency files like deps.edn and plugin manifests - - create-checksum-file: - filename: .BACKEND-CHECKSUMS - find-args: ". -type f -name '*.clj' -or -name '*.cljc' -or -name '*.java' -or -name '*.edn' -or -name '*.yaml' -or -name sample-dataset.db.mv.db" - # .SCRIPTS-DEPS-CHECKSUMS is all the deps.edn files inside ./bin - - create-checksum-file: - filename: .SCRIPTS-DEPS-CHECKSUMS - find-args: "bin -type f -name 'deps.edn'" - # .MODULES-CHECKSUMS is every Clojure source file in the modules/ directory as well as plugin manifests - - create-checksum-file: - filename: .MODULES-CHECKSUMS - find-args: "./modules -type f -name '*.clj' -or -name metabase-plugin.yaml" - - run: - name: Save last git commit message to .COMMIT - command: git log -1 > .COMMIT - - run: - name: Determine what to do to .git directory - command: | - if [[ $CIRCLE_BRANCH == release* ]]; then - echo 'This is a release branch; preserving .git directory to determine version' - else - echo 'This is not a release branch; removing .git directory (not needed for tests)' - rm -rf /home/circleci/metabase/metabase/.git - fi - - - run: - name: Remove ./OSX directory (not needed for tests) - command: rm -rf /home/circleci/metabase/metabase/OSX - # .CACHE-PREFIX is described above in the Cache Keys section of this file - - run: - name: 'Create cache key prefix .CACHE-PREFIX to bust caches if commit message includes [ci nocache]' - command: | - if [[ `cat .COMMIT` == *"[ci nocache]"* ]]; then - echo 'Commit message includes [ci nocache]; using cache-busting prefix' - echo '<< pipeline.id >>' > .CACHE-PREFIX - else - echo '' > .CACHE-PREFIX - fi - run: - name: 'If $CIRCLE_JOB is unset then bust the cache (see #24128 for details)' - command: | - if [ -z "$CIRCLE_JOB" ]; then - echo '$CIRCLE_JOB is unset. Using cache-busting prefix.' - echo '<< pipeline.id >>' > .CACHE-PREFIX - fi - - run: - name: Create static visualization js bundle - command: yarn build-static-viz - - run: - name: 'Check for branch name to bust caches if it is a release branch' - command: | - if [[ $CIRCLE_BRANCH == release* || $CIRCLE_BRANCH == master ]]; then - echo 'This is a release or master branch; using cache-busting prefix' - echo '<< pipeline.id >>' > .CACHE-PREFIX - fi - - run: - name: Make SSL certificates for Mongo available - command: >- - curl https://raw.githubusercontent.com/metabase/metabase-qa/master/dbs/mongo/certificates/metabase.crt - -o /home/circleci/metabase/metabase/test_resources/ssl/mongo/metabase.crt - https://raw.githubusercontent.com/metabase/metabase-qa/master/dbs/mongo/certificates/metabase.key - -o /home/circleci/metabase/metabase/test_resources/ssl/mongo/metabase.key - https://raw.githubusercontent.com/metabase/metabase-qa/master/dbs/mongo/certificates/metaca.crt - -o /home/circleci/metabase/metabase/test_resources/ssl/mongo/metaca.crt - - persist_to_workspace: - root: /home/circleci/ - paths: - - metabase/metabase - -######################################################################################################################## -# BACKEND # -######################################################################################################################## - - be-deps: - executor: builder - parameters: - <<: *Params - steps: - - attach-workspace - # This step is pretty slow, even with the cache, so only run it if deps.edn has changed - - run-on-change: - checksum: 'v5-{{ checksum "deps.edn" }}-{{ checksum ".SCRIPTS-DEPS-CHECKSUMS" }}' - steps: - - restore-be-deps-cache - - run: - name: Compile Java source file(s) - command: clojure -X:deps prep - - run: - name: Fetch dependencies - command: clojure -P -X:dev:ci:ee:ee-dev:drivers:drivers-dev - - run: - name: Fetch dependencies (./bin/build/build-mb) - command: cd /home/circleci/metabase/metabase/bin/build-mb && clojure -P -M:test - # Not sure why this is needed since you would think build-mb would fetch this stuff as well. It doesn't - # seem to fetch everything tho. :shrug: - - run: - name: Fetch dependencies (./bin/build/build-drivers) - command: cd /home/circleci/metabase/metabase/bin/build-drivers && clojure -P -M:test - - save_cache: - name: Cache backend dependencies - <<: *CacheKeyBackendDeps - paths: - - /home/circleci/.m2 - - /home/circleci/.gitlibs - - /home/circleci/metabase/metabase/java/target/classes - - /home/circleci/metabase/metabase/modules/drivers/sparksql/target/classes - - clojure: - parameters: - e: - type: executor - default: builder - before-steps: - type: steps - default: [] - clojure-args: - type: string - after-steps: - type: steps - default: [] - skip-when-no-change: - type: boolean - default: false - java-version: - type: string - default: "" - version: - type: string - default: "" - <<: *Params - executor: << parameters.e >> - steps: - - attach-workspace - - when: - condition: << parameters.skip-when-no-change >> - steps: - - run-on-change: - checksum: '{{ checksum ".BACKEND-CHECKSUMS" }}' - steps: - - run-clojure-command: - before-steps: << parameters.before-steps >> - clojure-args: << parameters.clojure-args >> - after-steps: << parameters.after-steps >> - edition: << parameters.edition >> - - unless: - condition: << parameters.skip-when-no-change >> - steps: - - run-clojure-command: - before-steps: << parameters.before-steps >> - clojure-args: << parameters.clojure-args >> - after-steps: << parameters.after-steps >> - edition: << parameters.edition >> - - be-linter-reflection-warnings: - executor: builder - steps: - - attach-workspace - - run-on-change: - checksum: '{{ checksum ".BACKEND-CHECKSUMS" }}-{{ checksum "bin/reflection-linter" }}' - steps: - - restore-be-deps-cache - - run: - name: Run reflection warnings checker - command: ./bin/reflection-linter - no_output_timeout: 15m - - test-driver: - parameters: - e: - type: executor - default: builder - driver: - type: string - timeout: - type: string - default: 20m - before-steps: - type: steps - default: [] - after-steps: - type: steps - default: [] - description: - type: string - default: "" - extra-env: - type: string - default: "" - test-args: - type: string - default: "" - version: - type: string - default: "" - executor: << parameters.e >> - steps: - - attach-workspace - - run-on-change: - checksum: '{{ checksum ".BACKEND-CHECKSUMS" }}' - skip-job-if-commit-message-includes-ci-quick: true - steps: - - restore-be-deps-cache - - steps: << parameters.before-steps >> - - run: - name: Test << parameters.driver >> driver << parameters.description >> - environment: - DRIVERS: << parameters.driver >> - command: > - << parameters.extra-env >> clojure -X:dev:ci:ee:ee-dev:drivers:drivers-dev:test - << parameters.test-args >> - no_output_timeout: << parameters.timeout >> - - store_test_results: - path: /home/circleci/metabase/metabase/target/junit - - steps: << parameters.after-steps >> - -######################################################################################################################## -# WORKFLOWS # -######################################################################################################################## - -# `default_matrix` isn't a key that CircleCI uses, but this form lets us reuse the matrix: block -default_matrix: &Matrix - matrix: - parameters: - edition: ["ee", "oss"] - -workflows: - version: 2 - build: - jobs: - - checkout - - - be-deps: - requires: - - checkout - - - clojure: - matrix: - parameters: - edition: ["ee"] - java-version: ["java-11"] - name: be-tests-<< matrix.java-version >>-<< matrix.edition >> - requires: - - be-deps - e: << matrix.java-version >> - clojure-args: -X:dev:ci:test - skip-when-no-change: true - - - clojure: - name: be-linter-cloverage - requires: - - be-deps - # TODO FIXME - clojure-args: -X:dev:ee:ee-dev:test:cloverage - after-steps: - - run: - name: Upload code coverage to codecov.io - command: bash <(curl -s https://codecov.io/bash) -F back-end - - skip-when-no-change: true - - - test-driver: - matrix: - parameters: - driver: ["bigquery-cloud-sdk", "googleanalytics", "sqlite"] - name: be-tests-<< matrix.driver >>-ee - requires: - - be-tests-java-11-ee - driver: << matrix.driver >> - - - test-driver: - matrix: - parameters: - driver: ["sqlserver", "druid"] - name: be-tests-<< matrix.driver >>-ee - requires: - - be-tests-java-11-ee - e: << matrix.driver >> - driver: << matrix.driver >> - - - test-driver: - name: be-google-related-drivers-classpath-test - requires: - - be-tests-java-11-ee - driver: googleanalytics,bigquery-cloud-sdk - test-args: >- - :only "[metabase.query-processor-test.expressions-test metabase.driver.google-test - metabase.driver.googleanalytics-test]" - - - test-driver: - matrix: - parameters: - version: ["mongo-4-0", "mongo-5-0", "mongo-latest", "mongo-4-0-ssl", "mongo-5-0-ssl"] - name: be-tests-<< matrix.version >>-ee - description: "(<< matrix.version >>)" - requires: - - be-tests-java-11-ee - e: << matrix.version >> - driver: mongo - - - test-driver: - matrix: - parameters: - version: ["mysql-5-7", "mariadb-10-2", "mariadb-latest"] - name: be-tests-<< matrix.version >>-ee - description: "(<< matrix.version >>)" - requires: - - be-tests-java-11-ee - e: - name: << matrix.version >> - driver: mysql - - - test-driver: - name: be-tests-mysql-latest-ee - description: "(MySQL latest)" - requires: - - be-tests-java-11-ee - e: - name: mysql-latest - driver: mysql - # set up env vars for something named "MYSQL_SSL" to run MySQL SSL tests verifying connectivity with PEM cert - # they are deliberately given a different name to prevent them from affecting the regular test run against - # the configured MySQL instance, but there is one particular test (mysql-connect-with-ssl-and-pem-cert-test) - # that overrides the MB_MYSQL_TEST_* values with them - # the MYSQL_RDS_SSL_INSTANCE vars are secret and/or changeable, so they are defined in the CircleCI settings - timeout: 30m - extra-env: >- - MB_MYSQL_SSL_TEST_HOST=$MYSQL_RDS_SSL_INSTANCE_HOST - MB_MYSQL_SSL_TEST_SSL=true - MB_MYSQL_SSL_TEST_ADDITIONAL_OPTIONS='verifyServerCertificate=true' - MB_MYSQL_SSL_TEST_SSL_CERT="$(cat /home/circleci/metabase/metabase/resources/certificates/rds-combined-ca-bundle.pem)" - MB_MYSQL_SSL_TEST_USER=metabase - MB_MYSQL_SSL_TEST_PASSWORD=$MYSQL_RDS_SSL_INSTANCE_PASSWORD - - - test-driver: - name: be-tests-oracle-ee - requires: - - be-tests-java-11-ee - before-steps: - - fetch-jdbc-driver: - source: ORACLE_JDBC_JAR - dest: ojdbc8.jar - - run: - name: Ensure truststore file - command: ls /home/circleci/metabase/metabase/resources/certificates/rds_root_ca_truststore.jks - driver: oracle - extra-env: >- - MB_ORACLE_SSL_TEST_SSL=true - MB_ORACLE_SSL_TEST_PORT=2484 - MB_ORACLE_SSL_TEST_SSL_USE_TRUSTSTORE=true - MB_ORACLE_SSL_TEST_SSL_TRUSTSTORE_PATH=/home/circleci/metabase/metabase/resources/certificates/rds_root_ca_truststore.jks - MB_ORACLE_SSL_TEST_SSL_TRUSTSTORE_OPTIONS=local - MB_ORACLE_SSL_TEST_SSL_TRUSTSTORE_PASSWORD_VALUE=metabase - - - test-driver: - name: be-tests-postgres-ee - description: "(9.6)" - requires: - - be-tests-java-11-ee - e: postgres-9-6 - driver: postgres - - - test-driver: - name: be-tests-postgres-latest-ee - description: "(Latest)" - requires: - - be-tests-java-11-ee - e: postgres-latest - driver: postgres - extra-env: >- - MB_POSTGRES_SSL_TEST_SSL=true - MB_POSTGRES_SSL_TEST_SSL_MODE=verify-full - MB_POSTGRES_SSL_TEST_SSL_ROOT_CERT_PATH=/home/circleci/metabase/metabase/test-resources/certificates/us-east-2-bundle.pem - - - test-driver: - name: be-tests-presto-ee - requires: - - be-tests-java-11-ee - e: presto-186 - before-steps: - - wait-for-port: - port: 8080 - driver: presto - - - test-driver: - name: be-tests-presto-jdbc-ee - requires: - - be-tests-java-11-ee - e: presto-jdbc-env # specific env for running Presto JDBC tests (newer Presto version, SSL, etc.) - before-steps: - - wait-for-port: - port: 8443 - - run: - name: Create temp cacerts file based on bundled JDK one - command: cp $JAVA_HOME/lib/security/cacerts /tmp/cacerts-with-presto-ssl.jks - - run: - name: Capture Presto server self signed CA - command: | - while [[ ! -s /tmp/presto-ssl-ca.pem ]]; - do echo "Waiting to capture SSL CA" \ - && openssl s_client -connect localhost:8443 2>/dev/null /tmp/presto-ssl-ca.pem \ - && sleep 1; done - - run: - name: Convert Presto CA from PEM to DER - command: openssl x509 -outform der -in /tmp/presto-ssl-ca.pem -out /tmp/presto-ssl-ca.der - - run: - name: Add write permission on cacerts file - command: chmod u+w /tmp/cacerts-with-presto-ssl.jks - - run: - name: Import Presto CA into temp cacerts file - command: | - keytool -noprompt -import -alias presto -keystore /tmp/cacerts-with-presto-ssl.jks \ - -storepass changeit -file /tmp/presto-ssl-ca.der -trustcacerts - after-steps: - - run: - name: Capture max memory usage - command: cat /sys/fs/cgroup/memory/memory.max_usage_in_bytes - when: always - driver: presto-jdbc - - - test-driver: - name: be-tests-redshift-ee - requires: - - be-tests-java-11-ee - driver: redshift - timeout: 15m - - - test-driver: - name: be-tests-snowflake-ee - requires: - - be-tests-java-11-ee - driver: snowflake - timeout: 115m - - - test-driver: - name: be-tests-sparksql-ee - requires: - - be-tests-java-11-ee - e: sparksql - before-steps: - - wait-for-port: - port: 10000 - driver: sparksql - - - test-driver: - name: be-tests-vertica-ee - requires: - - be-tests-java-11-ee - e: vertica - before-steps: - - fetch-jdbc-driver: - source: VERTICA_JDBC_JAR - dest: vertica-jdbc-7.1.2-0.jar - driver: vertica + name: echo + command: echo "Hello, World!" diff --git a/.clj-kondo/config.edn b/.clj-kondo/config.edn index 853a8d409d68..97b8ac9e32d1 100644 --- a/.clj-kondo/config.edn +++ b/.clj-kondo/config.edn @@ -224,13 +224,14 @@ metabase.driver.mongo.parameters mongo.params metabase.driver.mongo.query-processor mongo.qp metabase.driver.mongo.util mongo.util + metabase.driver.sql-jdbc.common sql-jdbc.common metabase.driver.sql-jdbc.connection sql-jdbc.conn metabase.driver.sql-jdbc.execute sql-jdbc.execute metabase.driver.sql-jdbc.execute.diagnostic sql-jdbc.execute.diagnostic metabase.driver.sql-jdbc.execute.legacy-impl sql-jdbc.legacy metabase.driver.sql-jdbc.execute.old-impl sql-jdbc.execute.old metabase.driver.sql-jdbc.sync sql-jdbc.sync - metabase.driver.sql-jdbc.sync.common sql-jdbc.common + metabase.driver.sql-jdbc.sync.common sql-jdbc.sync.common metabase.driver.sql-jdbc.sync.describe-database sql-jdbc.describe-database metabase.driver.sql-jdbc.sync.describe-table sql-jdbc.describe-table metabase.driver.sql-jdbc.sync.interface sql-jdbc.sync.interface @@ -401,6 +402,7 @@ metabase.integrations.ldap/with-ldap-connection clojure.core/fn metabase.mbql.schema.macros/defclause clj-kondo.lint-as/def-catch-all metabase.models.collection-test/with-collection-in-location clojure.core/let + metabase.models.json-migration/def-json-migration clj-kondo.lint-as/def-catch-all metabase.models.setting.multi-setting/define-multi-setting clojure.core/def metabase.models.setting/defsetting clj-kondo.lint-as/def-catch-all metabase.public-settings.premium-features/defenterprise clj-kondo.lint-as/def-catch-all @@ -572,7 +574,7 @@ metabase.query-processor-test.expressions-test/calculate-bird-scarcity hooks.metabase.query-processor-test.expressions-test/calculate-bird-scarcity metabase.query-processor-test.filter-test/count-with-filter-clause hooks.metabase.test.data/$ids metabase.query-processor.middleware.cache-test/with-mock-cache hooks.common/with-two-bindings - metabase.sample-database-test/with-temp-sample-database-db hooks.common/with-one-binding + metabase.sample-data-test/with-temp-sample-database-db hooks.common/with-one-binding metabase.test.data.datasets/test-drivers hooks.common/do* metabase.test.data.users/with-group hooks.common/let-one-with-optional-value metabase.test.data/$ids hooks.metabase.test.data/$ids diff --git a/.github/actions/find-squashed-commit/action.yml b/.github/actions/find-squashed-commit/action.yml index c8469da135c2..853df00c5913 100644 --- a/.github/actions/find-squashed-commit/action.yml +++ b/.github/actions/find-squashed-commit/action.yml @@ -22,7 +22,7 @@ runs: COMMIT=$(env -i git log $BASE_REF --grep="(#$PULL_REQUEST_NUMBER)" --format="%H") echo "commit SHA $COMMIT" - echo "::set-output name=commit::$COMMIT" + echo "commit=$COMMIT" >> $GITHUB_OUTPUT id: find-squashed-commit shell: bash env: diff --git a/.github/actions/test-driver/action.yml b/.github/actions/test-driver/action.yml new file mode 100644 index 000000000000..4162e98d01cb --- /dev/null +++ b/.github/actions/test-driver/action.yml @@ -0,0 +1,28 @@ +name: Test database driver +inputs: + junit-name: + required: true + default: 'driver' + test-args: + required: false + +runs: + using: "composite" + steps: + - name: Prepare front-end environment + uses: ./.github/actions/prepare-frontend + - name: Prepare back-end environment + uses: ./.github/actions/prepare-backend + - name: Build static viz frontend + run: yarn build-static-viz + shell: bash + - name: Test database driver + run: clojure -X:dev:ci:ee:ee-dev:drivers:drivers-dev:test ${{ inputs.test-args }} + shell: bash + - name: Publish Test Report (JUnit) + uses: dorny/test-reporter@v1 + if: always() + with: + path: 'target/junit/**/*_test.xml' + name: JUnit Test Report ${{ inputs.junit-name }} + reporter: java-junit diff --git a/.github/workflows/auto-backport.yml b/.github/workflows/auto-backport.yml index dfd68c67a0d0..3155339b846e 100644 --- a/.github/workflows/auto-backport.yml +++ b/.github/workflows/auto-backport.yml @@ -43,13 +43,13 @@ jobs: CONFLICTS=$(git ls-files -u | wc -l) if [ "$CONFLICTS" -gt 0 ]; then echo "Could not cherry pick because of a conflict" - echo "::set-output name=has-conflicts::true" + echo "has-conflicts=true" >> $GITHUB_OUTPUT git cherry-pick --abort git checkout master exit 0 fi - echo "::set-output name=has-conflicts::false" + echo "has-conflicts=false" >> $GITHUB_OUTPUT git checkout master git push -u origin ${BACKPORT_BRANCH} @@ -60,7 +60,7 @@ jobs: BACKPORT_PR_NUMBER=${BACKPORT_PR_URL##*/} - echo "::set-output name=backport_pr_number::$BACKPORT_PR_NUMBER" + echo "backport_pr_number=$BACKPORT_PR_NUMBER" >> $GITHUB_OUTPUT env: TARGET_BRANCH: ${{ steps.get_latest_release_branch.outputs.branch-name }} ORIGINAL_PULL_REQUEST_NUMBER: ${{ github.event.pull_request.number }} diff --git a/.github/workflows/backend-skipped-checks.yml b/.github/workflows/backend-skipped-checks.yml index 83f1fc1648dc..e1506b2d087b 100644 --- a/.github/workflows/backend-skipped-checks.yml +++ b/.github/workflows/backend-skipped-checks.yml @@ -4,12 +4,23 @@ name: Backend on: + push: + branches: + - 'master' + - 'release-**' + paths: + - "docs/**" + - "**.md" + # frontend and E2E tests + - "**/frontend/test/**" + - "**/frontend/**.unit.*" pull_request: paths: - "docs/**" - "**.md" - - "frontend/test/**" - - "enterprise/frontend/test/**" + # frontend and E2E tests + - "**/frontend/test/**" + - "**/frontend/**.unit.*" jobs: diff --git a/.github/workflows/backend.yml b/.github/workflows/backend.yml index 5dd66f163dd8..362509beca2d 100644 --- a/.github/workflows/backend.yml +++ b/.github/workflows/backend.yml @@ -5,15 +5,43 @@ on: branches: - 'master' - 'release-**' + paths-ignore: + - "docs/**" + - "**.md" + # frontend and E2E tests + - "**/frontend/test/**" + - "**/frontend/**.unit.*" pull_request: + types: [opened, synchronize, reopened, ready_for_review] paths-ignore: - "docs/**" - "**.md" - - "frontend/test/**" - - "enterprise/frontend/test/**" + # frontend and E2E tests + - "**/frontend/test/**" + - "**/frontend/**.unit.*" jobs: + be-linter-cloverage: + if: github.event.pull_request.draft == false + runs-on: ubuntu-20.04 + timeout-minutes: 60 + steps: + - uses: actions/checkout@v3 + - name: Build static viz frontend + run: yarn build-static-viz + - name: Prepare back-end environment + uses: ./.github/actions/prepare-backend + with: + m2-cache-key: 'cloverage' + - name: Collect the test coverage + run: clojure -X:dev:ee:ee-dev:test:cloverage + - name: Upload coverage to codecov.io + uses: codecov/codecov-action@v2 + with: + files: ./target/coverage/codecov.json + flags: back-end + be-linter-clj-kondo: runs-on: ubuntu-20.04 timeout-minutes: 10 @@ -65,6 +93,7 @@ jobs: /work/modules/drivers/presto-jdbc/test be-linter-eastwood: + if: github.event.pull_request.draft == false runs-on: ubuntu-20.04 timeout-minutes: 20 steps: @@ -76,10 +105,41 @@ jobs: - run: clojure -X:dev:ee:ee-dev:drivers:drivers-dev:test:eastwood name: Run Eastwood linter + # Because it's not possible to conditionally run only `java-11-ee` test in the draft mode, + # we have to extract that job manually here. Backend developers have requested that this + # test runs at all times to give them an early warning sign is something is broken. + be-tests-java-11-ee-pre-check: + if: github.event.pull_request.draft == true + runs-on: ubuntu-20.04 + name: be-tests-java-11-ee-pre-check + timeout-minutes: 25 + steps: + - uses: actions/checkout@v3 + - name: Prepare front-end environment + uses: ./.github/actions/prepare-frontend + - name: Prepare back-end environment + uses: ./.github/actions/prepare-backend + + - run: yarn install --frozen-lockfile --prefer-offline + - name: Build static viz frontend + run: yarn build-static-viz + + - name: Run tests + run: clojure -X:dev:ci:test:ee:ee-dev + + - name: Publish Test Report (JUnit) + uses: dorny/test-reporter@v1 + if: always() + with: + path: 'target/junit/**/*_test.xml' + name: JUnit Test Report be-tests-java-11-ee-pre-check + reporter: java-junit + be-tests: + if: github.event.pull_request.draft == false runs-on: ubuntu-20.04 name: be-tests-java-${{ matrix.java-version }}-${{ matrix.edition }} - timeout-minutes: 20 + timeout-minutes: 25 strategy: fail-fast: false matrix: @@ -96,25 +156,30 @@ jobs: - name: Build static viz frontend run: yarn build-static-viz - - name: Compile Java source file(s) - run: clojure -X:deps prep - - name: Compile driver AOT namespaces - working-directory: modules/drivers - run: clojure -X:deps prep - - name: Fetch dependencies - run: clojure -P -X:dev:ci:ee:ee-dev:drivers:drivers-dev - - name: Fetch dependencies (./bin/build/build-mb) - working-directory: bin/build-mb - run: clojure -P -M:test - - name: Fetch dependencies (./bin/build/build-drivers) - working-directory: bin/build-drivers - run: clojure -P -M:test - - name: Run tests run: clojure -X:dev:ci:test:${{ matrix.edition }}:${{ matrix.edition }}-dev + - name: Publish Test Report (JUnit) - uses: mikepenz/action-junit-report@v2 + uses: dorny/test-reporter@v1 if: always() with: - report_paths: 'target/junit/**/*_test.xml' - check_name: JUnit Test Report be-tests-java-${{ matrix.java-version }}-${{ matrix.edition }} + path: 'target/junit/**/*_test.xml' + name: JUnit Test Report be-tests-java-${{ matrix.java-version }}-${{ matrix.edition }} + reporter: java-junit + + # checks that all the namespaces we actually ship can be compiled, without any dependencies that we don't ship (such + # as `:dev` dependencies). See #27009 for more context. + be-check: + if: github.event.pull_request.draft == false + runs-on: ubuntu-20.04 + name: be-check-java-${{ matrix.java-version }} + timeout-minutes: 10 + strategy: + matrix: + java-version: [11, 17, 19] + steps: + - uses: actions/checkout@v3 + - name: Prepare backend + uses: ./.github/actions/prepare-backend + - name: Check namespaces + run: clojure -M:ee:drivers:check diff --git a/.github/workflows/backport.yml b/.github/workflows/backport.yml index 4eb8ea05c006..2285d20b57f1 100644 --- a/.github/workflows/backport.yml +++ b/.github/workflows/backport.yml @@ -117,7 +117,7 @@ jobs: BACKPORT_PR_NUMBER=${BACKPORT_PR_URL##*/} - echo "::set-output name=backport_pr_number::$BACKPORT_PR_NUMBER" + echo "backport_pr_number=$BACKPORT_PR_NUMBER" >> $GITHUB_OUTPUT echo "New PR has been created" fi env: diff --git a/.github/workflows/drivers-skipped-checks.yml b/.github/workflows/drivers-skipped-checks.yml new file mode 100644 index 000000000000..91b6c034da94 --- /dev/null +++ b/.github/workflows/drivers-skipped-checks.yml @@ -0,0 +1,175 @@ +# Required checks with path filtering rules will block pull requests from merging if they change only the excluded files. +# This is a workaround to allow the PR to be merged. +# https://docs.github.com/en/repositories/configuring-branches-and-merges-in-your-repository/defining-the-mergeability-of-pull-requests/troubleshooting-required-status-checks#handling-skipped-but-required-checks +name: Driver Tests + +on: + push: + branches: + - 'master' + - 'release-**' + paths: + - "docs/**" + - "**.md" + # frontend and E2E tests + - "**/frontend/test/**" + - "**/frontend/**.unit.*" + pull_request: + paths: + - "docs/**" + - "**.md" + # frontend and E2E tests + - "**/frontend/test/**" + - "**/frontend/**.unit.*" + +jobs: + + be-tests-athena-ee: + runs-on: ubuntu-20.04 + steps: + - run: | + echo "Didn't run due to conditional filtering" + + be-tests-bigquerycloud-sdk-ee: + runs-on: ubuntu-20.04 + steps: + - run: | + echo "Didn't run due to conditional filtering" + + be-tests-druid-ee: + runs-on: ubuntu-20.04 + steps: + - run: | + echo "Didn't run due to conditional filtering" + + be-tests-googleanalytics-ee: + runs-on: ubuntu-20.04 + steps: + - run: | + echo "Didn't run due to conditional filtering" + + be-tests-google-related-classpath-ee: + runs-on: ubuntu-20.04 + steps: + - run: | + echo "Didn't run due to conditional filtering" + + be-tests-mariadb-10-2-ee: + runs-on: ubuntu-20.04 + steps: + - run: | + echo "Didn't run due to conditional filtering" + + be-tests-mariadb-latest-ee: + runs-on: ubuntu-20.04 + steps: + - run: | + echo "Didn't run due to conditional filtering" + + be-tests-mongo-4-0-ee: + runs-on: ubuntu-20.04 + steps: + - run: | + echo "Didn't run due to conditional filtering" + + be-tests-mongo-4-0-ssl-ee: + runs-on: ubuntu-20.04 + steps: + - run: | + echo "Didn't run due to conditional filtering" + + be-tests-mongo-5-0-ee: + runs-on: ubuntu-20.04 + steps: + - run: | + echo "Didn't run due to conditional filtering" + + be-tests-mongo-5-0-ssl-ee: + runs-on: ubuntu-20.04 + steps: + - run: | + echo "Didn't run due to conditional filtering" + + be-tests-mongo-latest-ee: + runs-on: ubuntu-20.04 + steps: + - run: | + echo "Didn't run due to conditional filtering" + + be-tests-mysql-5-7-ee: + runs-on: ubuntu-20.04 + steps: + - run: | + echo "Didn't run due to conditional filtering" + + be-tests-mysql-latest-ee: + runs-on: ubuntu-20.04 + steps: + - run: | + echo "Didn't run due to conditional filtering" + + be-tests-oracle-ee: + runs-on: ubuntu-20.04 + steps: + - run: | + echo "Didn't run due to conditional filtering" + + be-tests-postgres-ee: + runs-on: ubuntu-20.04 + steps: + - run: | + echo "Didn't run due to conditional filtering" + + be-tests-postgres-latest-ee: + runs-on: ubuntu-20.04 + steps: + - run: | + echo "Didn't run due to conditional filtering" + + be-tests-presto-186-ee: + runs-on: ubuntu-20.04 + steps: + - run: | + echo "Didn't run due to conditional filtering" + + be-tests-presto-jdbc-ee: + runs-on: ubuntu-20.04 + steps: + - run: | + echo "Didn't run due to conditional filtering" + + be-tests-redshift-jdbc-ee: + runs-on: ubuntu-20.04 + steps: + - run: | + echo "Didn't run due to conditional filtering" + + be-tests-snowflake-jdbc-ee: + runs-on: ubuntu-20.04 + steps: + - run: | + echo "Didn't run due to conditional filtering" + + be-tests-sparksql-ee: + runs-on: ubuntu-20.04 + steps: + - run: | + echo "Didn't run due to conditional filtering" + + be-tests-sqlite-ee: + runs-on: ubuntu-20.04 + steps: + - run: | + echo "Didn't run due to conditional filtering" + + be-tests-sqlserver-ee: + runs-on: ubuntu-20.04 + steps: + - run: | + echo "Didn't run due to conditional filtering" + + be-tests-vertica-ee: + runs-on: ubuntu-20.04 + steps: + - run: | + echo "Didn't run due to conditional filtering" diff --git a/.github/workflows/drivers.yml b/.github/workflows/drivers.yml new file mode 100644 index 000000000000..c37929d14e98 --- /dev/null +++ b/.github/workflows/drivers.yml @@ -0,0 +1,643 @@ +name: Driver Tests + +on: + push: + branches: + - 'master' + - 'release-**' + paths-ignore: + - "docs/**" + - "**.md" + # frontend and E2E tests + - "**/frontend/test/**" + - "**/frontend/**.unit.*" + pull_request: + types: [opened, synchronize, reopened, ready_for_review] + paths-ignore: + - "docs/**" + - "**.md" + # frontend and E2E tests + - "**/frontend/test/**" + - "**/frontend/**.unit.*" + +concurrency: + group: ${{ github.head_ref || github.run_id}} + cancel-in-progress: true + +jobs: + + be-tests-athena-ee: + if: github.event.pull_request.draft == false + runs-on: ubuntu-20.04 + timeout-minutes: 60 + env: + CI: 'true' + DRIVERS: athena + MB_ATHENA_TEST_REGION: us-east-1 + MB_ATHENA_TEST_ACCESS_KEY: ${{ secrets.MB_ATHENA_TEST_ACCESS_KEY }} + MB_ATHENA_TEST_SECRET_KEY: ${{ secrets.MB_ATHENA_TEST_SECRET_KEY }} + MB_ATHENA_TEST_S3_STAGING_DIR: ${{ secrets.MB_ATHENA_TEST_S3_STAGING_DIR }} + steps: + - uses: actions/checkout@v3 + - name: Test Athena driver + uses: ./.github/actions/test-driver + with: + junit-name: 'be-tests-athena-ee' + + be-tests-bigquery-cloud-sdk-ee: + if: github.event.pull_request.draft == false + runs-on: ubuntu-20.04 + timeout-minutes: 60 + env: + CI: 'true' + DRIVERS: bigquery-cloud-sdk + MB_BIGQUERY_TEST_PROJECT_ID: ${{ secrets.BIGQUERY_TEST_PROJECT_ID }} + MB_BIGQUERY_TEST_CLIENT_ID: ${{ secrets.MB_BIGQUERY_TEST_CLIENT_ID }} + MB_BIGQUERY_TEST_CLIENT_SECRET: ${{ secrets.MB_BIGQUERY_TEST_CLIENT_SECRET }} + MB_BIGQUERY_TEST_ACCESS_TOKEN: ${{ secrets.MB_BIGQUERY_TEST_ACCESS_TOKEN }} + MB_BIGQUERY_TEST_REFRESH_TOKEN: ${{ secrets.MB_BIGQUERY_TEST_REFRESH_TOKEN }} + MB_BIGQUERY_CLOUD_SDK_TEST_SERVICE_ACCOUNT_JSON: ${{ secrets.MB_BIGQUERY_CLOUD_SDK_TEST_SERVICE_ACCOUNT_JSON }} + steps: + - uses: actions/checkout@v3 + - name: Test BigQuery Cloud SDK driver + uses: ./.github/actions/test-driver + with: + junit-name: 'be-tests-bigquery-cloud-sdk-ee' + + be-tests-druid-ee: + if: github.event.pull_request.draft == false + runs-on: buildjet-2vcpu-ubuntu-2004 + timeout-minutes: 60 + env: + CI: 'true' + DRIVERS: druid + services: + druid: + image: metabase/druid:0.20.2 + ports: + - "8082:8082" + env: + CLUSTER_SIZE: nano-quickstart + steps: + - uses: actions/checkout@v3 + - name: Test Druid driver + uses: ./.github/actions/test-driver + with: + junit-name: 'be-tests-druid-ee' + + be-tests-googleanalytics-ee: + if: github.event.pull_request.draft == false + runs-on: ubuntu-20.04 + timeout-minutes: 60 + env: + CI: 'true' + DRIVERS: googleanalytics + steps: + - uses: actions/checkout@v3 + - name: Test Google Analytics driver + uses: ./.github/actions/test-driver + with: + junit-name: 'be-tests-googleanalytics-ee' + + be-tests-google-related-classpath-ee: + if: github.event.pull_request.draft == false + runs-on: ubuntu-20.04 + timeout-minutes: 60 + strategy: + matrix: + driver: ['googleanalytics', 'bigquery-cloud-sdk'] + env: + CI: 'true' + DRIVERS: ${{ matrix.driver }} + MB_BIGQUERY_TEST_PROJECT_ID: ${{ secrets.BIGQUERY_TEST_PROJECT_ID }} + MB_BIGQUERY_TEST_CLIENT_ID: ${{ secrets.MB_BIGQUERY_TEST_CLIENT_ID }} + MB_BIGQUERY_TEST_CLIENT_SECRET: ${{ secrets.MB_BIGQUERY_TEST_CLIENT_SECRET }} + MB_BIGQUERY_TEST_ACCESS_TOKEN: ${{ secrets.MB_BIGQUERY_TEST_ACCESS_TOKEN }} + MB_BIGQUERY_TEST_REFRESH_TOKEN: ${{ secrets.MB_BIGQUERY_TEST_REFRESH_TOKEN }} + MB_BIGQUERY_CLOUD_SDK_TEST_SERVICE_ACCOUNT_JSON: ${{ secrets.MB_BIGQUERY_CLOUD_SDK_TEST_SERVICE_ACCOUNT_JSON }} + steps: + - uses: actions/checkout@v3 + - name: Test Google Related Classpath drivers + uses: ./.github/actions/test-driver + with: + junit-name: 'be-tests-${{ matrix.driver }}-classpath-ee' + test-args: ':only "[metabase.query-processor-test.expressions-test metabase.driver.google-test metabase.driver.googleanalytics-test]"' + + be-tests-mariadb-10-2-ee: + if: github.event.pull_request.draft == false + runs-on: ubuntu-20.04 + timeout-minutes: 60 + env: + CI: 'true' + DRIVERS: mysql + MB_DB_TYPE: mysql + MB_DB_HOST: localhost + MB_DB_PORT: 3306 + MB_DB_DBNAME: circle_test + MB_DB_USER: root + MB_MYSQL_TEST_USER: root + services: + mariadb: + image: circleci/mariadb:10.2.23 + ports: + - "3306:3306" + steps: + - uses: actions/checkout@v3 + - name: Test MariaDB driver (10.2) + uses: ./.github/actions/test-driver + with: + junit-name: 'be-tests-mariadb-10-2-ee' + + be-tests-mariadb-latest-ee: + if: github.event.pull_request.draft == false + runs-on: ubuntu-20.04 + timeout-minutes: 60 + env: + CI: 'true' + DRIVERS: mysql + MB_DB_TYPE: mysql + MB_DB_HOST: localhost + MB_DB_PORT: 3306 + MB_DB_DBNAME: circle_test + MB_DB_USER: root + MB_MYSQL_TEST_USER: root + services: + mariadb: + image: circleci/mariadb:latest + ports: + - "3306:3306" + steps: + - uses: actions/checkout@v3 + - name: Test MariaDB driver (latest) + uses: ./.github/actions/test-driver + with: + junit-name: 'be-tests-mariadb-latest-ee' + + be-tests-mongo-4-0-ee: + if: github.event.pull_request.draft == false + runs-on: ubuntu-20.04 + timeout-minutes: 60 + env: + CI: 'true' + DRIVERS: mongo + services: + mongodb: + image: metabase/qa-databases:mongo-sample-4.0 + ports: + - "27017:27017" + steps: + - uses: actions/checkout@v3 + - name: Test MongoDB driver (4.0) + uses: ./.github/actions/test-driver + with: + junit-name: 'be-tests-mongo-4-0-ee' + + be-tests-mongo-4-0-ssl-ee: + if: github.event.pull_request.draft == false + runs-on: ubuntu-20.04 + timeout-minutes: 60 + env: + CI: 'true' + DRIVERS: mongo + MB_MONGO_TEST_USER: metabase + MB_MONGO_TEST_PASSWORD: metasample123 + MB_TEST_MONGO_REQUIRES_SSL: true + steps: + - uses: actions/checkout@v3 + - name: Spin up Mongo docker container + run: docker run -d -p 27017:27017 --name metamongo metabase/qa-databases:mongo-sample-4.0 mongod --dbpath /data/db2/ --sslMode requireSSL --sslPEMKeyFile /etc/mongo/metamongo.pem --sslCAFile /etc/mongo/metaca.crt + - name: Wait until the port 27017 is ready + run: while ! nc -z localhost 27017; do sleep 1; done + timeout-minutes: 5 + - name: Make SSL certificates for Mongo available + run: | + curl https://raw.githubusercontent.com/metabase/metabase-qa/master/dbs/mongo/certificates/metabase.crt \ + -o ./test_resources/ssl/mongo/metabase.crt + + curl https://raw.githubusercontent.com/metabase/metabase-qa/master/dbs/mongo/certificates/metabase.key \ + -o ./test_resources/ssl/mongo/metabase.key + + curl https://raw.githubusercontent.com/metabase/metabase-qa/master/dbs/mongo/certificates/metaca.crt \ + -o ./test_resources/ssl/mongo/metaca.crt + + - name: Test MongoDB SSL driver (4.0) + uses: ./.github/actions/test-driver + with: + junit-name: 'be-tests-mongo-4-0-ee' + + be-tests-mongo-5-0-ee: + if: github.event.pull_request.draft == false + runs-on: ubuntu-20.04 + timeout-minutes: 60 + env: + CI: 'true' + DRIVERS: mongo + services: + mongodb: + image: metabase/qa-databases:mongo-sample-5.0 + ports: + - "27017:27017" + steps: + - uses: actions/checkout@v3 + - name: Test MongoDB driver (5.0) + uses: ./.github/actions/test-driver + with: + junit-name: 'be-tests-mongo-5-0-ee' + + be-tests-mongo-5-0-ssl-ee: + if: github.event.pull_request.draft == false + runs-on: ubuntu-20.04 + timeout-minutes: 60 + env: + CI: 'true' + DRIVERS: mongo + MB_MONGO_TEST_USER: metabase + MB_MONGO_TEST_PASSWORD: metasample123 + MB_TEST_MONGO_REQUIRES_SSL: true + steps: + - uses: actions/checkout@v3 + - name: Spin up Mongo docker container + run: docker run -d -p 27017:27017 --name metamongo metabase/qa-databases:mongo-sample-5.0 mongod --dbpath /data/db2/ --tlsMode requireTLS --tlsCertificateKeyFile /etc/mongo/metamongo.pem --tlsCAFile /etc/mongo/metaca.crt + - name: Wait until the port 27017 is ready + run: while ! nc -z localhost 27017; do sleep 1; done + timeout-minutes: 5 + - name: Make SSL certificates for Mongo available + run: | + curl https://raw.githubusercontent.com/metabase/metabase-qa/master/dbs/mongo/certificates/metabase.crt \ + -o ./test_resources/ssl/mongo/metabase.crt + + curl https://raw.githubusercontent.com/metabase/metabase-qa/master/dbs/mongo/certificates/metabase.key \ + -o ./test_resources/ssl/mongo/metabase.key + + curl https://raw.githubusercontent.com/metabase/metabase-qa/master/dbs/mongo/certificates/metaca.crt \ + -o ./test_resources/ssl/mongo/metaca.crt + + - name: Test MongoDB SSL driver (5.0) + uses: ./.github/actions/test-driver + with: + junit-name: 'be-tests-mongo-5-0-ee' + + be-tests-mongo-latest-ee: + if: github.event.pull_request.draft == false + runs-on: ubuntu-20.04 + timeout-minutes: 60 + env: + CI: 'true' + DRIVERS: mongo + services: + mongodb: + image: circleci/mongo:latest + ports: + - "27017:27017" + env: + MONGO_INITDB_ROOT_USERNAME: metabase + MONGO_INITDB_ROOT_PASSWORD: metasample123 + steps: + - uses: actions/checkout@v3 + - name: Test MongoDB driver (latest) + uses: ./.github/actions/test-driver + with: + junit-name: 'be-tests-mongo-latest-ee' + + be-tests-mysql-5-7-ee: + if: github.event.pull_request.draft == false + runs-on: ubuntu-20.04 + timeout-minutes: 60 + env: + CI: 'true' + DRIVERS: mysql + MB_DB_TYPE: mysql + MB_DB_HOST: localhost + MB_DB_PORT: 3306 + MB_DB_DBNAME: circle_test + MB_DB_USER: root + MB_MYSQL_TEST_USER: root + services: + mysql: + image: circleci/mysql:5.7.23 + ports: + - "3306:3306" + steps: + - uses: actions/checkout@v3 + - name: Test MySQL driver (5.7) + uses: ./.github/actions/test-driver + with: + junit-name: 'be-tests-mysql-5-7-ee' + + be-tests-mysql-latest-ee: + if: github.event.pull_request.draft == false + runs-on: ubuntu-20.04 + timeout-minutes: 60 + env: + CI: 'true' + DRIVERS: mysql + MB_DB_TYPE: mysql + MB_DB_HOST: localhost + MB_DB_PORT: 3306 + MB_DB_DBNAME: circle_test + MB_DB_USER: root + MB_MYSQL_TEST_USER: root + # set up env vars for something named "MYSQL_SSL" to run MySQL SSL tests verifying connectivity with PEM cert + # they are deliberately given a different name to prevent them from affecting the regular test run against + # the configured MySQL instance, but there is one particular test (mysql-connect-with-ssl-and-pem-cert-test) + # that overrides the MB_MYSQL_TEST_* values with them + # the MYSQL_RDS_SSL_INSTANCE vars are defined as secrets and can be altered + MB_MYSQL_SSL_TEST_HOST: ${{ secrets.MYSQL_RDS_SSL_INSTANCE_HOST }} + MB_MYSQL_SSL_TEST_SSL: true + MB_MYSQL_SSL_TEST_ADDITIONAL_OPTIONS: 'verifyServerCertificate=true' + # the contents of the ./resources/certificates/rds-combined-ca-bundle.pem file + MB_MYSQL_SSL_TEST_SSL_CERT: ${{ secrets.MB_MYSQL_SSL_TEST_SSL_CERT }} + MB_MYSQL_SSL_TEST_USER: metabase + MB_MYSQL_SSL_TEST_PASSWORD: ${{ secrets.MYSQL_RDS_SSL_INSTANCE_PASSWORD }} + services: + mysql: + image: circleci/mysql:latest + ports: + - "3306:3306" + steps: + - uses: actions/checkout@v3 + - name: Test MySQL driver (latest) + uses: ./.github/actions/test-driver + with: + junit-name: 'be-tests-mysql-latest-ee' + + be-tests-oracle-ee: + if: github.event.pull_request.draft == false + runs-on: ubuntu-20.04 + timeout-minutes: 60 + env: + CI: 'true' + DRIVERS: oracle + MB_ORACLE_TEST_USER: metabase + MB_ORACLE_TEST_PASSWORD: ${{ secrets.MB_ORACLE_TEST_PASSWORD }} + MB_ORACLE_TEST_SID: ORCL + MB_ORACLE_TEST_HOST: ${{ secrets.MB_ORACLE_TEST_HOST }} + MB_ORACLE_SSL_TEST_SSL: true + MB_ORACLE_SSL_TEST_PORT: 2484 + MB_ORACLE_SSL_TEST_SSL_USE_TRUSTSTORE: true + MB_ORACLE_SSL_TEST_SSL_TRUSTSTORE_PATH: './resources/certificates/rds_root_ca_truststore.jks' + MB_ORACLE_SSL_TEST_SSL_TRUSTSTORE_OPTIONS: local + MB_ORACLE_SSL_TEST_SSL_TRUSTSTORE_PASSWORD_VALUE: metabase + steps: + - uses: actions/checkout@v3 + - name: Ensure truststore file + run: ls ${{ env.MB_ORACLE_SSL_TEST_SSL_TRUSTSTORE_PATH }} + - name: Make plugins directory + run: mkdir plugins + - name: Fetch JDBC driver + run: wget --output-document=plugins/ojdbc8.jar ${{ secrets.ORACLE_JDBC_JAR }} + - name: Test Oracle driver + uses: ./.github/actions/test-driver + with: + junit-name: 'be-tests-oracle-ee' + + be-tests-postgres-ee: + if: github.event.pull_request.draft == false + runs-on: ubuntu-20.04 + timeout-minutes: 60 + env: + CI: 'true' + DRIVERS: postgres + MB_DB_TYPE: postgres + MB_DB_PORT: 5432 + MB_DB_HOST: localhost + MB_DB_DBNAME: circle_test + MB_DB_USER: circle_test + MB_POSTGRESQL_TEST_USER: circle_test + services: + postgres: + image: circleci/postgres:9.6-alpine + ports: + - "5432:5432" + env: + POSTGRES_USER: circle_test + POSTGRES_DB: circle_test + steps: + - uses: actions/checkout@v3 + - name: Test Postgres driver (9.6) + uses: ./.github/actions/test-driver + with: + junit-name: 'be-tests-postgres-ee' + + be-tests-postgres-latest-ee: + if: github.event.pull_request.draft == false + runs-on: ubuntu-20.04 + timeout-minutes: 60 + env: + CI: 'true' + DRIVERS: postgres + MB_DB_TYPE: postgres + MB_DB_PORT: 5432 + MB_DB_HOST: localhost + MB_DB_DBNAME: circle_test + MB_DB_USER: circle_test + MB_POSTGRESQL_TEST_USER: circle_test + MB_POSTGRES_SSL_TEST_SSL: true + MB_POSTGRES_SSL_TEST_SSL_MODE: verify-full + MB_POSTGRES_SSL_TEST_SSL_ROOT_CERT_PATH: 'test-resources/certificates/us-east-2-bundle.pem' + services: + postgres: + image: circleci/postgres:latest + ports: + - "5432:5432" + env: + POSTGRES_USER: circle_test + POSTGRES_DB: circle_test + POSTGRES_HOST_AUTH_METHOD: trust + steps: + - uses: actions/checkout@v3 + - name: Test Postgres driver (latest) + uses: ./.github/actions/test-driver + with: + junit-name: 'be-tests-postgres-latest-ee' + + be-tests-presto-186-ee: + if: github.event.pull_request.draft == false + runs-on: ubuntu-20.04 + timeout-minutes: 60 + env: + CI: 'true' + DRIVERS: presto + services: + presto: + image: metabase/presto-mb-ci:0.186 + ports: + - "8080:8080" + env: + JAVA_TOOL_OPTIONS: "-Xmx2g" + steps: + - uses: actions/checkout@v3 + - name: Test Presto 0.186 driver + uses: ./.github/actions/test-driver + with: + junit-name: 'be-tests-presto-186-ee' + + be-tests-presto-jdbc-ee: + if: github.event.pull_request.draft == false + runs-on: ubuntu-20.04 + timeout-minutes: 60 + env: + CI: 'true' + DRIVERS: presto-jdbc + MB_PRESTO_JDBC_TEST_CATALOG: test_data + MB_PRESTO_JDBC_TEST_HOST: localhost + MB_PRESTO_JDBC_TEST_PORT: 8443 + MB_PRESTO_JDBC_TEST_SSL: true + MB_PRESTO_JDBC_TEST_USER: metabase + MB_PRESTO_JDBC_TEST_PASSWORD: metabase + MB_ENABLE_PRESTO_JDBC_DRIVER: true + MB_PRESTO_JDBC_TEST_ADDITIONAL_OPTIONS: 'SSLTrustStorePath=/tmp/cacerts-with-presto-ssl.jks&SSLTrustStorePassword=changeit' + services: + presto: + image: metabase/presto-mb-ci:latest # version 0.254 + ports: + - "8443:8443" + env: + JAVA_TOOL_OPTIONS: "-Xmx2g" + steps: + - uses: actions/checkout@v3 + - name: Wait for port ${{ env.MB_PRESTO_JDBC_TEST_PORT }} to be ready + run: while ! nc -z localhost ${{ env.MB_PRESTO_JDBC_TEST_PORT }}; do sleep 0.1; done + timeout-minutes: 15 + - name: Create temp cacerts file based on bundled JDK one + run: cp $JAVA_HOME/lib/security/cacerts /tmp/cacerts-with-presto-ssl.jks + - name: Capture Presto server self signed CA + run: | + while [[ ! -s /tmp/presto-ssl-ca.pem ]]; + do echo "Waiting to capture SSL CA" \ + && openssl s_client -connect localhost:8443 2>/dev/null /tmp/presto-ssl-ca.pem \ + && sleep 1; done + - name: Convert Presto CA from PEM to DER + run: openssl x509 -outform der -in /tmp/presto-ssl-ca.pem -out /tmp/presto-ssl-ca.der + - name: Add write permission on cacerts file + run: chmod u+w /tmp/cacerts-with-presto-ssl.jks + - name: Import Presto CA into temp cacerts file + run: | + keytool -noprompt -import -alias presto -keystore /tmp/cacerts-with-presto-ssl.jks \ + -storepass changeit -file /tmp/presto-ssl-ca.der -trustcacerts + - name: Test Presto JDBC driver + uses: ./.github/actions/test-driver + with: + junit-name: 'be-tests-presto-jdbc-ee' + - name: Capture max memory usage + run: cat /sys/fs/cgroup/memory/memory.max_usage_in_bytes + + be-tests-redshift-ee: + if: github.event.pull_request.draft == false + runs-on: ubuntu-20.04 + timeout-minutes: 60 + env: + CI: 'true' + DRIVERS: redshift + MB_REDSHIFT_TEST_USER: metabase_ci + MB_REDSHIFT_TEST_DB: testdb + MB_REDSHIFT_TEST_HOST: ${{ secrets.MB_REDSHIFT_TEST_HOST }} + MB_REDSHIFT_TEST_PASSWORD: ${{ secrets.MB_REDSHIFT_TEST_PASSWORD }} + steps: + - uses: actions/checkout@v3 + - name: Test Redshift driver + uses: ./.github/actions/test-driver + with: + junit-name: 'be-tests-redshift-ee' + + be-tests-snowflake-ee: + if: github.event.pull_request.draft == false + runs-on: ubuntu-20.04 + timeout-minutes: 60 + env: + CI: 'true' + DRIVERS: snowflake + MB_SNOWFLAKE_TEST_USER: METABASE CI + MB_SNOWFLAKE_TEST_ACCOUNT: ${{ secrets.MB_SNOWFLAKE_TEST_ACCOUNT }} + MB_SNOWFLAKE_TEST_PASSWORD: ${{ secrets.MB_SNOWFLAKE_TEST_PASSWORD }} + MB_SNOWFLAKE_TEST_WAREHOUSE: ${{ secrets.MB_SNOWFLAKE_TEST_WAREHOUSE }} + MB_SNOWFLAKE_TEST_PK_USER: METABASE PK + MB_SNOWFLAKE_TEST_PK_PRIVATE_KEY: ${{ secrets.MB_SNOWFLAKE_TEST_PK_PRIVATE_KEY }} + steps: + - uses: actions/checkout@v3 + - name: Test Snowflake driver + uses: ./.github/actions/test-driver + with: + junit-name: 'be-tests-snowflake-ee' + + be-tests-sparksql-ee: + if: github.event.pull_request.draft == false + runs-on: buildjet-2vcpu-ubuntu-2004 + timeout-minutes: 60 + env: + CI: 'true' + DRIVERS: sparksql + services: + sparksql: + image: metabase/spark:3.2.1 + ports: + - "10000:10000" + steps: + - uses: actions/checkout@v3 + - name: Test Spark driver + uses: ./.github/actions/test-driver + with: + junit-name: 'be-tests-sparksql-ee' + + be-tests-sqlite-ee: + if: github.event.pull_request.draft == false + runs-on: ubuntu-20.04 + timeout-minutes: 60 + env: + CI: 'true' + DRIVERS: sqlite + steps: + - uses: actions/checkout@v3 + - name: Test SQLite driver + uses: ./.github/actions/test-driver + with: + junit-name: 'be-tests-sqlite-ee' + + be-tests-sqlserver-ee: + if: github.event.pull_request.draft == false + runs-on: ubuntu-20.04 + timeout-minutes: 60 + env: + CI: 'true' + DRIVERS: sqlserver + MB_SQLSERVER_TEST_HOST: localhost + MB_SQLSERVER_TEST_PASSWORD: 'P@ssw0rd' + MB_SQLSERVER_TEST_USER: SA + services: + sqlserver: + image: mcr.microsoft.com/mssql/server:2017-latest + ports: + - "1433:1433" + env: + ACCEPT_EULA: Y + SA_PASSWORD: 'P@ssw0rd' + MSSQL_MEMORY_LIMIT_MB: 1024 + steps: + - uses: actions/checkout@v3 + - name: Test MS SQL Server driver + uses: ./.github/actions/test-driver + with: + junit-name: 'be-tests-sqlserver-ee' + + be-tests-vertica-ee: + if: github.event.pull_request.draft == false + runs-on: ubuntu-20.04 + timeout-minutes: 60 + env: + CI: 'true' + DRIVERS: vertica + services: + vertica: + image: sumitchawla/vertica + ports: + - "5433:5433" + steps: + - uses: actions/checkout@v3 + - name: Make plugins directory + run: mkdir plugins + - name: Fetch JDBC driver + run: wget --output-document=plugins/vertica-jdbc-7.1.2-0.jar ${{ secrets.VERTICA_JDBC_JAR }} + - name: Test Vertica driver + uses: ./.github/actions/test-driver + with: + junit-name: 'be-tests-vertica-ee' diff --git a/.github/workflows/e2e-main.yml b/.github/workflows/e2e-main.yml index 26c1b1005d44..3509d2b39230 100644 --- a/.github/workflows/e2e-main.yml +++ b/.github/workflows/e2e-main.yml @@ -10,6 +10,7 @@ on: - "**.md" - ".circleci/**" - "**.unit.spec.*" + - "**_test.clj" jobs: @@ -43,7 +44,7 @@ jobs: runs-on: ubuntu-20.04 timeout-minutes: 45 needs: build - name: e2e-tests-${{ matrix.folder }}-${{ matrix.edition }} + name: e2e-tests-${{ matrix.folder }}${{ matrix.context }}-${{ matrix.edition }} env: MB_EDITION: ${{ matrix.edition }} DISPLAY: "" @@ -51,14 +52,12 @@ jobs: MB_PREMIUM_EMBEDDING_TOKEN: ${{ secrets.ENTERPRISE_TOKEN }} MB_SNOWPLOW_AVAILABLE: true MB_SNOWPLOW_URL: "http://localhost:9090" # Snowplow micro - RECORDING_ENABLED: ${{ secrets.CURRENTS_KEY }} - CYPRESS_DEPLOYSENTINEL_KEY: ${{ secrets.CYPRESS_DEPLOYSENTINEL_KEY }} ELECTRON_EXTRA_LAUNCH_ARGS: '--remote-debugging-port=40500' # deploysentinel strategy: fail-fast: false matrix: java-version: [11] - edition: [oss, ee] + edition: [ee] folder: - "admin" - "binning" @@ -79,6 +78,10 @@ jobs: - "question" - "sharing" - "visualizations" + include: + - edition: oss + context: grep + java-version: 11 services: maildev: image: maildev/maildev:1.1.0 @@ -118,6 +121,11 @@ jobs: password: ${{ secrets.DOCKERHUB_TOKEN }} steps: - uses: actions/checkout@v3 + # Test runs will be recorded using Deploysentinel on `master` only + - name: Set conditional ENVs + if: github.ref == 'refs/heads/master' + run: | + echo "CYPRESS_DEPLOYSENTINEL_KEY=${{ secrets.CYPRESS_DEPLOYSENTINEL_KEY }}" >> $GITHUB_ENV - name: Prepare front-end environment uses: ./.github/actions/prepare-frontend - name: Prepare JDK ${{ matrix.java-version }} @@ -139,19 +147,18 @@ jobs: jar xf target/uberjar/metabase.jar version.properties mv version.properties resources/ - - name: Run Cypress tests on ${{ matrix.folder }} - Master Branch - if: github.ref == 'refs/heads/master' && env.RECORDING_ENABLED != null + - name: Run OSS-specific Cypress tests + if: matrix.edition == 'oss' run: | yarn run test-cypress-run \ - --folder ${{ matrix.folder }} \ - --record --key ${{ secrets.CURRENTS_KEY }} \ - --group ${{ matrix.folder }}-${{ matrix.edition }} \ - --ci-build-id "${{ github.run_id }}-${{ github.run_attempt }}" + --env grepTags=@OSS \ + --spec './frontend/test/metabase/scenarios/**/*.cy.spec.js' env: TERM: xterm - - name: Run Cypress tests on ${{ matrix.folder }} - if: ${{ github.ref != 'refs/heads/master' }} + # These are EE-specific and version-agnostic tests + - name: Run Cypress E2E tests against EE uberjar + if: matrix.edition == 'ee' run: | yarn run test-cypress-run \ --folder ${{ matrix.folder }} @@ -162,7 +169,7 @@ jobs: uses: actions/upload-artifact@v2 if: failure() with: - name: cypress-artifacts-${{ matrix.folder }}-${{ matrix.edition }} + name: cypress-recording-${{ matrix.folder }}${{ matrix.context }}-${{ matrix.edition }} path: | ./cypress ./logs/test.log diff --git a/.github/workflows/e2e-tests-skipped-checks.yml b/.github/workflows/e2e-tests-skipped-checks.yml index 3726003cb772..a3a0f96dbe4e 100644 --- a/.github/workflows/e2e-tests-skipped-checks.yml +++ b/.github/workflows/e2e-tests-skipped-checks.yml @@ -11,6 +11,7 @@ on: - "**.md" - ".circleci/**" - "**.unit.spec.*" + - "**_test.clj" jobs: e2e-tests: diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml index 681be2211865..c9ae0a65e059 100644 --- a/.github/workflows/e2e-tests.yml +++ b/.github/workflows/e2e-tests.yml @@ -8,6 +8,7 @@ on: - "**.md" - ".circleci/**" - "**.unit.spec.*" + - "**_test.clj" concurrency: group: ${{ github.workflow }}-${{ github.ref }} @@ -54,6 +55,7 @@ jobs: MB_PREMIUM_EMBEDDING_TOKEN: ${{ secrets.ENTERPRISE_TOKEN }} MB_SNOWPLOW_AVAILABLE: true MB_SNOWPLOW_URL: "http://localhost:9090" # Snowplow micro + ELECTRON_EXTRA_LAUNCH_ARGS: '--remote-debugging-port=40500' # deploysentinel strategy: fail-fast: false matrix: @@ -122,6 +124,11 @@ jobs: password: ${{ secrets.DOCKERHUB_TOKEN }} steps: - uses: actions/checkout@v3 + - name: Set conditional ENVs + run: | + if [[ ${{ github.event.pull_request.base.ref }} != release* ]]; then + echo "CYPRESS_DEPLOYSENTINEL_KEY=${{ secrets.CYPRESS_DEPLOYSENTINEL_KEY }}" >> $GITHUB_ENV + fi - name: Prepare front-end environment uses: ./.github/actions/prepare-frontend - name: Prepare JDK ${{ matrix.java-version }} @@ -147,7 +154,7 @@ jobs: if: matrix.edition == 'oss' run: | yarn run test-cypress-run \ - --env grepTags=@OSS,grepFilterSpecs=true \ + --env grepTags=@OSS \ --spec './frontend/test/metabase/scenarios/**/*.cy.spec.js' env: TERM: xterm diff --git a/.github/workflows/frontend-skipped-checks.yml b/.github/workflows/frontend-skipped-checks.yml index 86cf5c9cb9a3..76a40c5e32b9 100644 --- a/.github/workflows/frontend-skipped-checks.yml +++ b/.github/workflows/frontend-skipped-checks.yml @@ -1,11 +1,37 @@ +# Required checks with path filtering rules will block pull requests from merging if they change only the excluded files. +# This is a workaround to allow the PR to be merged. +# https://docs.github.com/en/repositories/configuring-branches-and-merges-in-your-repository/defining-the-mergeability-of-pull-requests/troubleshooting-required-status-checks#handling-skipped-but-required-checks name: Frontend on: + push: + branches: + - 'master' + - 'release-**' + paths: + # documentation + - "docs/**" + - "**.md" + # backend + - "enterprise/backend/**" + - "src/**" + - "test/**" + - "*modules/**" # modules/, test_modules/ + # E2E + - "**.cy.*.js" # .cy.spec.js, .cy.snap.js + - "frontend/test/__support__/e2e/**" + - "frontend/test/__runner__/*cypress*" pull_request: paths: + # documentation - "docs/**" - "**.md" - - "**_test.clj" + # backend + - "enterprise/backend/**" + - "src/**" + - "test/**" + - "*modules/**" # modules/, test_modules/ + # E2E - "**.cy.*.js" # .cy.spec.js, .cy.snap.js - "frontend/test/__support__/e2e/**" - "frontend/test/__runner__/*cypress*" diff --git a/.github/workflows/frontend.yml b/.github/workflows/frontend.yml index 057c46e129da..85bccab7645d 100644 --- a/.github/workflows/frontend.yml +++ b/.github/workflows/frontend.yml @@ -5,11 +5,31 @@ on: branches: - 'master' - 'release-**' + paths-ignore: + # documentation + - "docs/**" + - "**.md" + # backend + - "enterprise/backend/**" + - "src/**" + - "test/**" + - "*modules/**" # modules/, test_modules/ + # E2E + - "**.cy.*.js" # .cy.spec.js, .cy.snap.js + - "frontend/test/__support__/e2e/**" + - "frontend/test/__runner__/*cypress*" pull_request: + types: [opened, synchronize, reopened, ready_for_review] paths-ignore: + # documentation - "docs/**" - "**.md" - - "**_test.clj" + # backend + - "enterprise/backend/**" + - "src/**" + - "test/**" + - "*modules/**" # modules/, test_modules/ + # E2E - "**.cy.*.js" # .cy.spec.js, .cy.snap.js - "frontend/test/__support__/e2e/**" - "frontend/test/__runner__/*cypress*" @@ -46,6 +66,7 @@ jobs: name: Check types fe-tests-unit: + if: github.event.pull_request.draft == false runs-on: buildjet-2vcpu-ubuntu-2004 timeout-minutes: 20 steps: @@ -61,6 +82,7 @@ jobs: flags: front-end fe-tests-timezones: + if: github.event.pull_request.draft == false runs-on: ubuntu-20.04 timeout-minutes: 14 steps: @@ -71,6 +93,7 @@ jobs: name: Run frontend timezones tests fe-chromatic: + if: github.event.pull_request.draft == false runs-on: ubuntu-20.04 steps: - name: Checkout repository diff --git a/.github/workflows/pre-release.yml b/.github/workflows/pre-release.yml new file mode 100644 index 000000000000..2e09e6c6563f --- /dev/null +++ b/.github/workflows/pre-release.yml @@ -0,0 +1,171 @@ +name: Pre-release [WIP] + +on: + workflow_dispatch: + inputs: + commit: + description: 'A full-length commit SHA-1 hash' + required: true + +env: + MAX_HASH_LENGTH: 8 + CUSTOM_REPO: ${{ secrets.CUSTOM_RELEASE_REPO }} + +jobs: + build: + if: ${{ github.repository }} != 'metabase/metabase' + name: Build Metabase ${{ matrix.edition }} @${{ github.event.inputs.commit }} + runs-on: ubuntu-20.04 + timeout-minutes: 40 + strategy: + matrix: + edition: [oss, ee] + env: + MB_EDITION: ${{ matrix.edition }} + INTERACTIVE: false + steps: + - name: Fail early if custom docker relaese repo is missing + if: ${{ env.CUSTOM_REPO == null }} + run: exit 1 + - name: Check out the code + uses: actions/checkout@v3 + with: + ref: ${{ github.event.inputs.commit }} + - name: Prepare front-end environment + uses: ./.github/actions/prepare-frontend + - name: Prepare back-end environment + uses: ./.github/actions/prepare-backend + - name: Build + run: ./bin/build + - name: Prepare uberjar artifact + uses: ./.github/actions/prepare-uberjar-artifact + + check-uberjar-health: + runs-on: ubuntu-20.04 + name: Is ${{ matrix.edition }} (java ${{ matrix.java-version }}) healthy? + needs: build + timeout-minutes: 10 + strategy: + matrix: + edition: [oss, ee] + java-version: [11, 17] + steps: + - name: Prepare JRE (Java Run-time Environment) + uses: actions/setup-java@v3 + with: + java-package: jre + java-version: ${{ matrix.java-version }} + distribution: 'temurin' + - run: java -version + - uses: actions/download-artifact@v2 + name: Retrieve uberjar artifact + with: + name: metabase-${{ matrix.edition }}-uberjar + - name: Launch uberjar (and keep it running) + run: java -jar ./target/uberjar/metabase.jar & + - name: Wait for Metabase to start + run: while ! curl -s 'http://localhost:3000/api/health' | grep '{"status":"ok"}'; do sleep 1; done + + containerize: + runs-on: ubuntu-20.04 + needs: check-uberjar-health + timeout-minutes: 15 + strategy: + matrix: + edition: [oss, ee] + services: + registry: + image: registry:2 + ports: + - 5000:5000 + steps: + - name: Check out the code + uses: actions/checkout@v3 + with: + ref: ${{ github.event.inputs.commit }} + - name: Truncate commit hash + run: | + commit_id=${{ github.event.inputs.commit }} + truncated_hash=${commit_id:0:${{ env.MAX_HASH_LENGTH }}} + + echo "COMMIT_IDENTIFIER=$truncated_hash" >> $GITHUB_ENV + shell: bash + - uses: actions/download-artifact@v3 + name: Retrieve uberjar artifact + with: + name: metabase-${{ matrix.edition }}-uberjar + - name: Move the Uberjar to the context dir + run: mv ./target/uberjar/metabase.jar bin/docker/. + - name: Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v2 + with: + driver-opts: network=host + - name: Build ${{ matrix.edition }} container + uses: docker/build-push-action@v3 + with: + context: bin/docker/. + platforms: linux/amd64 + network: host + tags: localhost:5000/local-metabase:${{ env.COMMIT_IDENTIFIER }}-${{ matrix.edition }} + no-cache: true + push: true + + - name: Launch container + run: docker run --rm -dp 3000:3000 localhost:5000/local-metabase:${{ env.COMMIT_IDENTIFIER }}-${{ matrix.edition }} + timeout-minutes: 5 + - name: Wait for Metabase to start + run: while ! curl -s 'http://localhost:3000/api/health' | grep '{"status":"ok"}'; do sleep 1; done + timeout-minutes: 3 + + - name: Login to Docker Hub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKERHUB_RELEASE_USERNAME }} + password: ${{ secrets.DOCKERHUB_RELEASE_TOKEN }} + - name: Determine the target Docker Hub repository + run: | + echo "DOCKERHUB_REPO=${{ github.repository_owner }}/${{ env.CUSTOM_REPO }}" >> $GITHUB_ENV + echo "IMAGE_NAME=${{ env.COMMIT_IDENTIFIER }}-${{ matrix.edition }}" >> $GITHUB_ENV + - name: Retag and push container image to Docker Hub + run: | + echo "Pushing container image ${{ env.IMAGE_NAME}} to ${{ env.DOCKERHUB_REPO }} ..." + docker tag localhost:5000/local-metabase:${{ env.IMAGE_NAME }} ${{ env.DOCKERHUB_REPO }}:${{ env.IMAGE_NAME }} + docker push ${{ env.DOCKERHUB_REPO }}:${{ env.IMAGE_NAME }} + echo "Finished!" + + verify-docker-pull: + runs-on: ubuntu-20.04 + needs: containerize + timeout-minutes: 15 + strategy: + matrix: + edition: [oss, ee] + steps: + - name: Truncate commit hash + run: | + commit_id=${{ github.event.inputs.commit }} + truncated_hash=${commit_id:0:${{ env.MAX_HASH_LENGTH }}} + + echo "COMMIT_IDENTIFIER=$truncated_hash" >> $GITHUB_ENV + shell: bash + - name: Login to Docker Hub # authenticated, to avoid being rate-throttled + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKERHUB_RELEASE_USERNAME }} + password: ${{ secrets.DOCKERHUB_RELEASE_TOKEN }} + - name: Determine the container image to pull + run: | + echo "DOCKERHUB_REPO=${{ github.repository_owner }}/${{ env.CUSTOM_REPO }}" >> $GITHUB_ENV + echo "IMAGE_NAME=${{ env.COMMIT_IDENTIFIER }}-${{ matrix.edition }}" >> $GITHUB_ENV + - name: Pull the container image + run: | + echo "Pulling container image ${{ env.DOCKERHUB_REPO }}:${{ env.IMAGE_NAME }} ..." + docker pull ${{ env.DOCKERHUB_REPO }}:${{ env.IMAGE_NAME }} + echo "Successful!" + - name: Launch container + run: docker run --rm -dp 3000:3000 ${{ env.DOCKERHUB_REPO }}:${{ env.IMAGE_NAME }} + timeout-minutes: 5 + - name: Wait for Metabase to start + run: while ! curl -s 'http://localhost:3000/api/health' | grep '{"status":"ok"}'; do sleep 1; done + timeout-minutes: 3 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index bbfde2af77f9..3bf3c838253f 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,51 +1,49 @@ name: Release [WIP] on: - workflow_dispatch: - inputs: - commit: - description: 'A commit hash' - required: true - oss-tag: - description: 'OSS version tag' - required: true - ee-tag: - description: 'EE version tag' - required: true + push: + tags: + - 'v*' jobs: - build: - name: Build Metabase ${{ matrix.edition }} @${{ github.event.inputs.commit }} + + download-uberjar: runs-on: ubuntu-20.04 - timeout-minutes: 40 - strategy: - matrix: - edition: [oss, ee] - env: - MB_EDITION: ${{ matrix.edition }} - INTERACTIVE: false + timeout-minutes: 10 steps: - - name: Check out the code - uses: actions/checkout@v3 + - name: Download Uberjar for ${{ github.ref_name }} + run: | + JAR_DOWNLOAD_URL=https://downloads.metabase.com/${{ github.ref_name }}/metabase.jar + if [[ ${{ github.ref_name }} == v1* ]]; then + JAR_DOWNLOAD_URL=https://downloads.metabase.com/enterprise/${{ github.ref_name }}/metabase.jar + fi + echo $JAR_DOWNLOAD_URL > url.txt + echo "----- Downloading Uberjar from $JAR_DOWNLOAD_URL -----" + curl -OL $JAR_DOWNLOAD_URL + stat ./metabase.jar + date | tee timestamp + - name: Verify that this is a valid JAR file + run: file --mime-type ./metabase.jar | grep "application/zip" + - name: Reveal its version.properties + run: jar xf metabase.jar version.properties && cat version.properties + - name: Calculate SHA256 checksum + run: sha256sum ./metabase.jar | tee SHA256.sum + - name: Upload Uberjar as artifact + uses: actions/upload-artifact@v3 with: - ref: ${{ github.event.inputs.commit }} - - name: Prepare front-end environment - uses: ./.github/actions/prepare-frontend - - name: Prepare back-end environment - uses: ./.github/actions/prepare-backend - - name: Build - run: ./bin/build - - name: Prepare uberjar artifact - uses: ./.github/actions/prepare-uberjar-artifact + name: metabase-uberjar-${{ github.ref_name }} + path: | + ./metabase.jar + ./url.txt + ./timestamp + ./SHA256.sum - check-uberjar-health: + check-uberjar: runs-on: ubuntu-20.04 - name: Is ${{ matrix.edition }} (java ${{ matrix.java-version }}) healthy? - needs: build + needs: download-uberjar timeout-minutes: 10 strategy: matrix: - edition: [oss, ee] java-version: [11, 17] steps: - name: Prepare JRE (Java Run-time Environment) @@ -54,62 +52,60 @@ jobs: java-package: jre java-version: ${{ matrix.java-version }} distribution: 'temurin' - - run: java -version - - uses: actions/download-artifact@v2 - name: Retrieve uberjar artifact + - uses: actions/download-artifact@v3 + name: Retrieve previously downloaded Uberjar with: - name: metabase-${{ matrix.edition }}-uberjar - - name: Launch uberjar (and keep it running) - run: java -jar ./target/uberjar/metabase.jar & + name: metabase-uberjar-${{ github.ref_name }} + - name: Reveal its version.properties + run: jar xf metabase.jar version.properties && cat version.properties + - name: Display when and where it was downloaded + run: | + cat timestamp + cat url.txt + - name: Show the checksum + run: cat SHA256.sum + - name: Launch Metabase Uberjar (and keep it running) + run: java -jar ./metabase.jar & - name: Wait for Metabase to start - run: while ! curl -s 'http://localhost:3000/api/health' | grep '{"status":"ok"}'; do sleep 1; done + run: while ! curl -s localhost:3000/api/health; do sleep 1; done + timeout-minutes: 3 + - name: Check API health + run: curl -s localhost:3000/api/health containerize: runs-on: ubuntu-20.04 - needs: check-uberjar-health + needs: check-uberjar timeout-minutes: 15 - strategy: - matrix: - edition: [oss, ee] services: registry: image: registry:2 ports: - 5000:5000 steps: - - name: Set the image tag based on the edition - run: | - if [[ ${{ matrix.edition }} == ee ]]; then - echo "IMAGE_TAG=${{ github.event.inputs.ee-tag }}" >> $GITHUB_ENV - else - echo "IMAGE_TAG=${{ github.event.inputs.oss-tag }}" >> $GITHUB_ENV - fi - uses: actions/checkout@v3 + - uses: actions/download-artifact@v3 + name: Retrieve previously downloaded Uberjar with: - ref: ${{ github.event.inputs.commit }} - - uses: actions/download-artifact@v2 - name: Retrieve uberjar artifact - with: - name: metabase-${{ matrix.edition }}-uberjar + name: metabase-uberjar-${{ github.ref_name }} - name: Move the Uberjar to the context dir - run: mv ./target/uberjar/metabase.jar bin/docker/. + run: mv ./metabase.jar bin/docker/. - name: Set up Docker Buildx id: buildx - uses: docker/setup-buildx-action@v1 + uses: docker/setup-buildx-action@v2 with: driver-opts: network=host - name: Build ${{ matrix.edition }} container - uses: docker/build-push-action@v2 + uses: docker/build-push-action@v3 with: context: bin/docker/. platforms: linux/amd64 network: host - tags: localhost:5000/local-metabase:${{ env.IMAGE_TAG }} + tags: localhost:5000/local-metabase:${{ github.ref_name }} no-cache: true push: true - name: Launch container - run: docker run --rm -dp 3000:3000 localhost:5000/local-metabase:${{ env.IMAGE_TAG }} + run: docker run --rm -dp 3000:3000 localhost:5000/local-metabase:${{ github.ref_name }} timeout-minutes: 5 - name: Wait for Metabase to start run: while ! curl -s 'http://localhost:3000/api/health' | grep '{"status":"ok"}'; do sleep 1; done @@ -117,25 +113,51 @@ jobs: - name: Determine the target Docker Hub repository run: | - if [[ ${{ github.repository }} == 'metabase/metabase' ]]; then - if [[ ${{ matrix.edition }} == ee ]]; then - echo "Metabase EE: image ${{ env.IMAGE_TAG }} is going to be pushed to metabase/test-metabase-enterprise" - echo "DOCKERHUB_REPO=metabase/test-metabase-enterprise" >> $GITHUB_ENV - else - echo "Metabase OSS: image ${{ env.IMAGE_TAG }} is going to be pushed to metabase/test-metabase" - echo "DOCKERHUB_REPO=metabase/test-metabase" >> $GITHUB_ENV - fi + if [[ ${{ github.ref_name }} == v1* ]]; then + echo "Metabase EE: image is going to be pushed to ${{ github.repository_owner }}/metabase-enterprise" + echo "DOCKERHUB_REPO=${{ github.repository_owner }}/metabase-enterprise" >> $GITHUB_ENV else - echo "DOCKERHUB_REPO=${{ github.repository_owner }}/${{ secrets.CUSTOM_RELEASE_REPO }}" >> $GITHUB_ENV + echo "Metabase OSS: image is going to be pushed to ${{ github.repository_owner }}/metabase" + echo "DOCKERHUB_REPO=${{ github.repository_owner }}/metabase" >> $GITHUB_ENV fi + - name: Login to Docker Hub - uses: docker/login-action@v1 + uses: docker/login-action@v2 with: username: ${{ secrets.DOCKERHUB_RELEASE_USERNAME }} password: ${{ secrets.DOCKERHUB_RELEASE_TOKEN }} - - name: Retag and push container image to Metabase Docker Hub + - name: Retag and push container image to Docker Hub run: | - echo "Pushing ${{ env.IMAGE_TAG }} to ${{ env.DOCKERHUB_REPO }} ..." - docker tag localhost:5000/local-metabase:${{ env.IMAGE_TAG }} ${{ env.DOCKERHUB_REPO }}:${{ env.IMAGE_TAG }} - docker push ${{ env.DOCKERHUB_REPO }}:${{ env.IMAGE_TAG }} + echo "Pushing ${{ github.ref_name }} to ${{ env.DOCKERHUB_REPO }} ..." + docker tag localhost:5000/local-metabase:${{ github.ref_name }} ${{ env.DOCKERHUB_REPO }}:${{ github.ref_name }} + docker push ${{ env.DOCKERHUB_REPO }}:${{ github.ref_name }} echo "Finished!" + + verify-docker-pull: + runs-on: ubuntu-20.04 + needs: containerize + timeout-minutes: 15 + steps: + - name: Login to Docker Hub # authenticated, to avoid being rate-throttled + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKERHUB_RELEASE_USERNAME }} + password: ${{ secrets.DOCKERHUB_RELEASE_TOKEN }} + - name: Determine the container image to pull + run: | + if [[ ${{ github.ref_name }} == v1* ]]; then + echo "DOCKERHUB_REPO=${{ github.repository_owner }}/metabase-enterprise" >> $GITHUB_ENV + else + echo "DOCKERHUB_REPO=${{ github.repository_owner }}/metabase" >> $GITHUB_ENV + fi + - name: Pull the container image + run: | + echo "Pulling container image ${{ env.DOCKERHUB_REPO }}:${{ github.ref_name }} ..." + docker pull ${{ env.DOCKERHUB_REPO }}:${{ github.ref_name }} + echo "Successful!" + - name: Launch container + run: docker run --rm -dp 3000:3000 ${{ env.DOCKERHUB_REPO }}:${{ github.ref_name }} + timeout-minutes: 5 + - name: Wait for Metabase to start + run: while ! curl -s 'http://localhost:3000/api/health' | grep '{"status":"ok"}'; do sleep 1; done + timeout-minutes: 3 diff --git a/.github/workflows/rerun-workflows.yml b/.github/workflows/rerun-workflows.yml new file mode 100644 index 000000000000..1796bfbd1315 --- /dev/null +++ b/.github/workflows/rerun-workflows.yml @@ -0,0 +1,31 @@ +name: Rerun Flaky Workflows + +on: + workflow_run: + workflows: [Backend, Driver Tests, E2E Tests, Frontend] + types: [completed] + branches: [master, 'release-**'] + +jobs: + rerun-on-failure: + runs-on: ubuntu-latest + if: ${{ github.event.workflow_run.conclusion }} == 'failure' + steps: + - uses: actions/github-script@v6 + with: + script: | + const MAX_ATTEMPTS = 2; + const ATTEMPT = ${{ github.event.workflow_run.run_attempt }}; + + if (ATTEMPT <= MAX_ATTEMPTS) { + console.log("Rerruning..."); + + github.rest.actions.reRunWorkflowFailedJobs({ + owner: context.repo.owner, + repo: context.repo.repo, + run_id: ${{ github.event.workflow_run.id }}, + }); + } else { + console.log("Rerunning didn't help!"); + console.log("Please check workflow " + ${{ github.event.workflow_run.id }}); + } diff --git a/.github/workflows/uberjar.yml b/.github/workflows/uberjar.yml index c45ac6203341..a854abe17521 100644 --- a/.github/workflows/uberjar.yml +++ b/.github/workflows/uberjar.yml @@ -4,10 +4,11 @@ on: push: paths-ignore: - 'docs/**' - - 'frontend/test/**' - - 'enterprise/frontend/test/**' + - "**.md" + - '**/frontend/test/**' - ".**" - "test*" + - "**_test.clj" jobs: build: @@ -105,7 +106,7 @@ jobs: run: docker ps - name: Wait for Metabase to start and reach 100% health run: while ! curl -s 'http://localhost:3000/api/health' | grep '{"status":"ok"}'; do sleep 1; done - timeout-minutes: 1 + timeout-minutes: 3 - name: Login to Docker Hub uses: docker/login-action@v1 with: @@ -126,6 +127,8 @@ jobs: - name: Run Trivy vulnerability scanner if master or main (ee) if: ${{ (github.ref_name == 'master' || github.ref_name == 'main') && matrix.edition == 'ee' }} uses: aquasecurity/trivy-action@master + env: + TRIVY_OFFLINE_SCAN: true with: image-ref: docker.io/metabase/metabase-enterprise-head:latest format: sarif @@ -134,6 +137,8 @@ jobs: - name: Run Trivy vulnerability scanner if master or main (oss) if: ${{ (github.ref_name == 'master' || github.ref_name == 'main') && matrix.edition == 'oss' }} uses: aquasecurity/trivy-action@master + env: + TRIVY_OFFLINE_SCAN: true with: image-ref: docker.io/metabase/metabase-head:latest format: sarif @@ -142,6 +147,8 @@ jobs: - name: Run Trivy vulnerability scanner if dev branch if: ${{ !(startsWith(github.ref_name,'master') || startsWith(github.ref_name,'main') || startsWith(github.ref_name,'backport')) && matrix.edition == 'ee' }} uses: aquasecurity/trivy-action@master + env: + TRIVY_OFFLINE_SCAN: true with: image-ref: docker.io/metabase/metabase-dev:${{ steps.extract_branch.outputs.branch }} format: sarif diff --git a/.mlc_config.json b/.mlc_config.json index f7f9a643895d..6770ef90a187 100644 --- a/.mlc_config.json +++ b/.mlc_config.json @@ -2,14 +2,14 @@ "ignorePatterns": [ { "pattern": "^https://downloads.metabase.com" + }, + { + "pattern": "^https://docs.cypress.io/" } ], "timeout": "180s", "retryOn429": true, "retryCount": 5, "fallbackRetryDelay": "30s", - "aliveStatusCodes": [ - 200, - 206 - ] + "aliveStatusCodes": [200, 206] } diff --git a/Dockerfile b/Dockerfile index 78162c045ace..ff8a24cb6d9a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -38,6 +38,9 @@ RUN apk add -U bash ttf-dejavu fontconfig curl java-cacerts && \ COPY --from=builder /home/circleci/target/uberjar/metabase.jar /app/ COPY bin/docker/run_metabase.sh /app/ +RUN curl -L https://github.com/ClickHouse/metabase-clickhouse-driver/releases/download/1.0.1/clickhouse.metabase-driver.jar > /plugins/clickhouse.metabase-driver.jar +RUN chmod 744 /plugins/clickhouse.metabase-driver.jar + # expose our default runtime port EXPOSE 3000 diff --git a/bin/build-drivers/deps.edn b/bin/build-drivers/deps.edn index 72ba9ef3edf1..aa816924fe21 100644 --- a/bin/build-drivers/deps.edn +++ b/bin/build-drivers/deps.edn @@ -18,6 +18,13 @@ metabase/metabase-core {:local/root "../.."} metabase/driver-modules {:local/root "../../modules/drivers"}} + ;; These are needed for the Athena and Redshift drivers in order to build them. Maven repos from subprojects do not + ;; get copied over -- see + ;; https://ask.clojure.org/index.php/10726/deps-manifest-dependencies-respect-repos-dependent-project + :mvn/repos + {"athena" {:url "https://s3.amazonaws.com/maven-athena"} + "redshift" {:url "https://s3.amazonaws.com/redshift-maven-repository/release"}} + :jvm-opts ["-XX:-OmitStackTraceInFastThrow"] diff --git a/bin/build-mb/deps.edn b/bin/build-mb/deps.edn index a43859f42428..8b4f8b752a62 100644 --- a/bin/build-mb/deps.edn +++ b/bin/build-mb/deps.edn @@ -9,6 +9,13 @@ ;; value currently used in tools.build but top level since we directly depend on it org.apache.maven/maven-model {:mvn/version "3.8.4"}} + ;; These are needed for the Athena and Redshift drivers in order to build them. Maven repos from subprojects do not + ;; get copied over -- see + ;; https://ask.clojure.org/index.php/10726/deps-manifest-dependencies-respect-repos-dependent-project + :mvn/repos + {"athena" {:url "https://s3.amazonaws.com/maven-athena"} + "redshift" {:url "https://s3.amazonaws.com/redshift-maven-repository/release"}} + :aliases {:test {:extra-paths ["test"] :extra-deps {com.cognitect/test-runner {:git/url "https://github.com/cognitect-labs/test-runner.git" diff --git a/bin/i18n/import-po-from-poeditor b/bin/i18n/import-po-from-poeditor index c5d81da761b4..033208393e0c 100755 --- a/bin/i18n/import-po-from-poeditor +++ b/bin/i18n/import-po-from-poeditor @@ -61,43 +61,29 @@ async function main(args) { } } -async function promXhr(uri, options) { - const xhr = new XMLHttpRequest(); - return new Promise((resolve, reject) => { - xhr.open(options.method, uri, true); - for (const headerName in options.headers) { - xhr.setRequestHeader(headerName, options.headers[headerName]); - } - xhr.send(options.body); - xhr.onload = () => { - const responseObject = { - status: xhr.status, - response: xhr.responseText, - }; - xhr.status >= 200 && xhr.status <= 299 - ? resolve(responseObject) - : reject(responseObject); - }; - }); -} - // simple API client for poeditor function poeditor(command, params = {}) { - const uri = url.format({ - protocol: "https", - hostname: "api.poeditor.com", - pathname: `/v2/${command}`, - }); const query = { api_token: POEDITOR_API_TOKEN, id: POEDITOR_PROJECT_ID, ...params, }; - return promXhr(uri, { - method: "POST", - headers: { "Content-Type": "application/x-www-form-urlencoded" }, - body: url.format({ query }).replace(/^\?/, ""), - }).then(res => res.json()); + return new Promise((resolve, reject) => { + const req = https.request({ + hostname: "api.poeditor.com", + path: `/v2/${command}`, + method: "POST", + headers: { "Content-Type": "application/x-www-form-urlencoded" }, + }, res => { + const chunks = []; + res.on("data", chunk => { chunks.push(chunk); }); + + res.on("end", () => { resolve(JSON.parse(Buffer.concat(chunks))) }); + }) + req.on('error',reject); + req.write(url.format({ query }).replace(/^\?/, "")); + req.end(); + }) } async function getExistingLanguages() { diff --git a/bin/lint-migrations-file/src/change_set/strict.clj b/bin/lint-migrations-file/src/change_set/strict.clj index 84772d420f4f..8504e5d70c8e 100644 --- a/bin/lint-migrations-file/src/change_set/strict.clj +++ b/bin/lint-migrations-file/src/change_set/strict.clj @@ -28,7 +28,54 @@ (apply distinct? (mapcat #(str/split (-> % :sql :dbms) #",") changes)))))) +(def change-types-supporting-rollback + ;; This set was generated with a little grep and awk from the docs here: + ;; https://docs.liquibase.com/workflows/liquibase-community/liquibase-auto-rollback.html + ;; + ;; If a new change type is introduced that supports automatic rollback, it should be added + ;; to this set. + #{:addCheckConstraint + :addColumn + :addDefaultValue + :addForeignKeyConstraint + :addLookupTable + :addNotNullConstraint + :addPrimaryKey + :addUniqueConstraint + :createIndex + :createSequence + :createSynonym + :createTable + :createView + :disableCheckConstraint + :disableTrigger + :dropNotNullConstraint + :enableCheckConstraint + :enableTrigger + :renameColumn + :renameSequence + :renameTable + :renameTrigger + :renameView}) + +(defn- major-version + "Returns major version from id string, e.g. 44 from \"v44.00-034\"" + [id-str] + (when (string? id-str) + (some-> (re-find #"\d+" id-str) Integer/parseInt))) + +(defn- rollback-present-when-required? + "Ensures rollback key is present when change type doesn't support auto rollback" + [{:keys [id changes] :as change-set}] + (or + (int? id) + (< (major-version id) 45) + (some change-types-supporting-rollback (mapcat keys changes)) + (contains? change-set :rollback))) + (s/def ::change-set - (s/merge - :change-set.common/change-set - (s/keys :req-un [::changes ::comment]))) + (s/and + rollback-present-when-required? + (s/merge + :change-set.common/change-set + (s/keys :req-un [::changes ::comment])))) diff --git a/bin/lint-migrations-file/test/lint_migrations_file_test.clj b/bin/lint-migrations-file/test/lint_migrations_file_test.clj index 6734d5481783..02693229df31 100644 --- a/bin/lint-migrations-file/test/lint_migrations_file_test.clj +++ b/bin/lint-migrations-file/test/lint_migrations_file_test.clj @@ -191,3 +191,21 @@ (mock-change-set :id "v42.00-001" :changes [(mock-add-column-changes :columns [(mock-column :type problem-type)])])))))))) + +(deftest require-rollback-test + (testing "change types with no automatic rollback support" + (testing "missing rollback key fails" + (is (thrown-with-msg? + clojure.lang.ExceptionInfo + #"rollback-present-when-required" + (validate (mock-change-set :id "v45.12-345" :changes [{:sql {:sql "select 1"}}]))))) + (testing "nil rollback is allowed" + (is (= :ok (validate (mock-change-set :id "v45.12-345" + :changes [{:sql {:sql "select 1"}}] + :rollback nil))))) + (testing "rollback values are allowed" + (is (= :ok (validate (mock-change-set :id "v45.12-345" + :changes [{:sql {:sql "select 1"}}] + :rollback {:sql {:sql "select 1"}})))))) + (testing "change types with automatic rollback support are allowed" + (is (= :ok (validate (mock-change-set :id "v45.12-345" :changes [(mock-add-column-changes)])))))) diff --git a/bin/release/deps.edn b/bin/release/deps.edn index cffa79de3fec..7e3ad1c5f7c2 100644 --- a/bin/release/deps.edn +++ b/bin/release/deps.edn @@ -9,6 +9,13 @@ org.flatland/ordered {:mvn/version "1.5.7"} stencil/stencil {:mvn/version "0.5.0"}} + ;; These are needed for the Athena and Redshift drivers in order to build them. Maven repos from subprojects do not + ;; get copied over -- see + ;; https://ask.clojure.org/index.php/10726/deps-manifest-dependencies-respect-repos-dependent-project + :mvn/repos + {"athena" {:url "https://s3.amazonaws.com/maven-athena"} + "redshift" {:url "https://s3.amazonaws.com/redshift-maven-repository/release"}} + :aliases {:test {:extra-paths ["test"] :extra-deps {com.cognitect/test-runner {:git/url "https://github.com/cognitect-labs/test-runner.git" diff --git a/bin/release/src/release.clj b/bin/release/src/release.clj index 4e4d41459cbe..395524bd9c6a 100644 --- a/bin/release/src/release.clj +++ b/bin/release/src/release.clj @@ -6,7 +6,6 @@ [release [check-prereqs :as check-prereqs] [common :as c] - [docker :as docker] [draft-release :as draft-release] [elastic-beanstalk :as eb] [git-tags :as git-tags] @@ -21,10 +20,8 @@ (def ^:private steps* (ordered-map/ordered-map :build-uberjar uberjar/build-uberjar! - :build-docker docker/build-docker-image! - :push-git-tags git-tags/push-tags! :upload-uberjar uberjar/upload-uberjar! - :push-docker-image docker/push-docker-image! + :push-git-tags git-tags/push-tags! :publish-draft-release draft-release/create-draft-release! :update-heroku-buildpack heroku/update-heroku-buildpack! :publish-elastic-beanstalk-artifacts eb/publish-elastic-beanstalk-artifacts! diff --git a/bin/release/src/release/docker.clj b/bin/release/src/release/docker.clj deleted file mode 100644 index 408209fb823a..000000000000 --- a/bin/release/src/release/docker.clj +++ /dev/null @@ -1,41 +0,0 @@ -(ns release.docker - "Code related to building, pushing, and validating new Docker images." - (:require [metabuild-common.core :as u] - [release.common :as c])) - -(defn build-docker-image! [] - (u/step "Build Docker image" - (let [docker-dir (u/filename c/root-directory "bin" "docker") - uberjar-path (u/filename docker-dir "metabase.jar")] - (u/delete-file-if-exists! uberjar-path) - (u/copy-file! (u/assert-file-exists c/uberjar-path) uberjar-path) - (u/assert-file-exists uberjar-path) - (u/sh "docker" "build" "--no-cache" "--pull" "-t" (c/docker-tag) docker-dir)))) - -(defn- validate-docker-image [] - (u/step "Validate Docker image" - (u/announce "TODO") - ;; image="$1" - ;; docker pull "$image" > /dev/null - ;; docker_hash=$(docker run --rm "$image" version | grep -Eo 'hash [0-9a-f]+' | awk '{ print $2 }') - ;; check-equals "docker: image tagged "$image" commit hash" "$tag_hash" "$docker_hash" - - ;; check-docker-image "$METABASE_DOCKER_REPO:v$VERSION" - )) - -(defn push-docker-image! [] - (u/step "Push Docker image" - (u/sh "docker" "push" (c/docker-tag)) - (let [latest-tag (str (c/docker-image-name) ":latest")] - (cond - (c/pre-release-version?) - (u/announce "Pre release version -- not pushing %s" latest-tag) - - (not (c/latest-version?)) - (u/announce "Version is not latest -- not pushing %s" latest-tag) - - :else - (u/step (format "Pushing tag %s" latest-tag) - (u/sh "docker" "tag" (c/docker-tag) latest-tag) - (u/sh "docker" "push" latest-tag))))) - (validate-docker-image)) diff --git a/codecov.yml b/codecov.yml index 623d717692ee..50f529dc2efd 100644 --- a/codecov.yml +++ b/codecov.yml @@ -1,6 +1,6 @@ codecov: bot: "codecov-io" - require_ci_to_pass: no + require_ci_to_pass: false coverage: status: diff --git a/deps.edn b/deps.edn index 00f28fa71a12..2a714f425072 100644 --- a/deps.edn +++ b/deps.edn @@ -32,7 +32,8 @@ com.draines/postal {:mvn/version "2.0.5"} ; SMTP library com.google.guava/guava {:mvn/version "31.0.1-jre"} ; dep for BigQuery, Spark, and GA. Require here rather than letting different dep versions stomp on each other — see comments on #9697 com.fasterxml.jackson.core/jackson-databind - {:mvn/version "2.13.2.2"} ; JSON processor used by snowplow-java-tracker (pinned version due to CVE-2020-36518) + {:mvn/version "2.13.4.2"} ; JSON processor used by snowplow-java-tracker (pinned version due to CVE-2020-36518) + com.fasterxml.woodstox/woodstox-core {:mvn/version "6.4.0"} ; trans dep of commons-codec (pinned version due to CVE-2022-40151) com.h2database/h2 {:mvn/version "1.4.197"} ; embedded SQL database com.snowplowanalytics/snowplow-java-tracker {:mvn/version "0.12.0" ; Snowplow analytics @@ -95,8 +96,8 @@ org.apache.poi/poi-ooxml {:mvn/version "5.2.2" :exclusions [org.bouncycastle/bcpkix-jdk15on org.bouncycastle/bcprov-jdk15on]} - org.apache.sshd/sshd-core {:mvn/version "2.9.1"} ; ssh tunneling and test server - org.apache.xmlgraphics/batik-all {:mvn/version "1.14"} ; SVG -> image + org.apache.sshd/sshd-core {:mvn/version "2.9.2"} ; ssh tunneling and test server + org.apache.xmlgraphics/batik-all {:mvn/version "1.16"} ; SVG -> image org.clojars.pntblnk/clj-ldap {:mvn/version "0.0.17"} ; LDAP client org.bouncycastle/bcpkix-jdk15on {:mvn/version "1.70"} ; Bouncy Castle crypto library -- explicit version of BC specified to resolve illegal reflective access errors org.bouncycastle/bcprov-jdk15on {:mvn/version "1.70"} @@ -107,12 +108,14 @@ org.clojure/core.match {:mvn/version "1.0.0"} org.clojure/core.memoize {:mvn/version "1.0.257"} ; useful FIFO, LRU, etc. caching mechanisms org.clojure/data.csv {:mvn/version "1.0.1"} ; CSV parsing / generation + org.clojure/data.xml {:mvn/version "0.0.8"} ; XML parsing / generation org.clojure/java.classpath {:mvn/version "1.0.0"} ; examine the Java classpath from Clojure programs org.clojure/java.jdbc {:mvn/version "0.7.12"} ; basic JDBC access from Clojure org.clojure/java.jmx {:mvn/version "1.0.0"} ; JMX bean library, for exporting diagnostic info org.clojure/math.combinatorics {:mvn/version "0.1.6"} ; combinatorics functions org.clojure/math.numeric-tower {:mvn/version "0.0.5"} ; math functions like `ceil` org.clojure/tools.logging {:mvn/version "1.2.4"} ; logging framework + org.clojure/tools.macro {:mvn/version "0.1.5"} ; local macros org.clojure/tools.namespace {:mvn/version "1.3.0"} org.clojure/tools.reader {:mvn/version "1.3.6"} org.clojure/tools.trace {:mvn/version "0.7.11"} ; function tracing @@ -128,7 +131,7 @@ org.slf4j/slf4j-api {:mvn/version "1.7.36"} ; abstraction for logging frameworks -- allows end user to plug in desired logging framework at deployment time org.tcrawley/dynapath {:mvn/version "1.1.0"} ; Dynamically add Jars (e.g. Oracle or Vertica) to classpath org.threeten/threeten-extra {:mvn/version "1.7.0"} ; extra Java 8 java.time classes like DayOfMonth and Quarter - org.yaml/snakeyaml {:mvn/version "1.30"} ; YAML parser + org.yaml/snakeyaml {:mvn/version "1.33"} ; YAML parser potemkin/potemkin {:mvn/version "0.4.5" ; utility macros & fns :exclusions [riddley/riddley]} pretty/pretty {:mvn/version "1.0.5"} ; protocol for defining how custom types should be pretty printed @@ -158,6 +161,20 @@ :paths ["src" "shared/src" "resources"] + ;; These are needed for the Athena and Redshift drivers if you are developing against them locally. If those drivers' + ;; dependencies are not included (i.e., if we don't have the `:drivers` profile), these repos are effectively + ;; ignored. + ;; + ;; 1. Maven repos from subprojects do not get copied over -- see + ;; https://ask.clojure.org/index.php/10726/deps-manifest-dependencies-respect-repos-dependent-project + ;; + ;; 2. You cannot include `:mvn/repos` inside of an alias -- see + ;; https://ask.clojure.org/index.php/12367/support-mvn-repos-inside-an-alias -- if we could, this could go in the + ;; `:drivers` alias instead. + :mvn/repos + {"athena" {:url "https://s3.amazonaws.com/maven-athena"} + "redshift" {:url "https://s3.amazonaws.com/redshift-maven-repository/release"}} + :aliases { ;;; Local Dev & test profiles @@ -271,7 +288,7 @@ ;; clojure -X:dev:ee:ee-dev:drivers:drivers-dev:test (for EE) :drivers-dev {:extra-paths - ["modules/drivers/bigquery/test" + ["modules/drivers/athena/test" "modules/drivers/bigquery-cloud-sdk/test" "modules/drivers/druid/test" "modules/drivers/googleanalytics/test" @@ -289,11 +306,33 @@ ;;; Linters - ;; clojure -M:dev:ee:ee-dev:drivers:drivers-dev:check + ;; clojure -M:ee:drivers:check + ;; + ;; checks that all the namespaces we actually ship can be compiled, without any dependencies that we don't + ;; ship (such as `:dev` dependencies). See #27009 for more context. :check {:extra-deps {athos/clj-check {:git/url "https://github.com/athos/clj-check.git" :sha "518d5a1cbfcd7c952f548e6dbfcb9a4a5faf9062"}} - :main-opts ["-m" "clj-check.check"]} + :main-opts ["-m" "clj-check.check" + "src" + "shared/src" + "enterprise/backend/src" + "modules/drivers/athena/src" + "modules/drivers/bigquery-cloud-sdk/src" + "modules/drivers/druid/src" + "modules/drivers/googleanalytics/src" + "modules/drivers/mongo/src" + "modules/drivers/oracle/src" + "modules/drivers/presto/src" + "modules/drivers/presto-common/src" + "modules/drivers/presto-jdbc/src" + "modules/drivers/redshift/src" + "modules/drivers/snowflake/src" + "modules/drivers/sparksql/src" + "modules/drivers/sqlite/src" + "modules/drivers/sqlserver/src" + "modules/drivers/vertica/src"] + :jvm-opts ["-Dclojure.main.report=stderr"]} ;; clojure -X:dev:ee:ee-dev:drivers:drivers-dev:test:eastwood :eastwood @@ -303,7 +342,7 @@ :source-paths ["src" "shared/src" "enterprise/backend/src" - "modules/drivers/bigquery/src" + "modules/drivers/athena/src" "modules/drivers/bigquery-cloud-sdk/src" "modules/drivers/druid/src" "modules/drivers/googleanalytics/src" diff --git a/dev/src/dev/render_png.clj b/dev/src/dev/render_png.clj index 730fc7c1468b..3a9871319617 100644 --- a/dev/src/dev/render_png.clj +++ b/dev/src/dev/render_png.clj @@ -72,9 +72,9 @@ (.write w html-str)) (.deleteOnExit tmp-file) (open tmp-file))) - + (comment - (render-card-to-png 1) + (render-card-to-png 1) ;; open viz in your browser (-> [["A" "B"] [1 2] diff --git a/docs/README.md b/docs/README.md index 9b19ceb789e3..8f965a63baa6 100644 --- a/docs/README.md +++ b/docs/README.md @@ -32,6 +32,7 @@ Metabase is a deep product with a lot of tools to simplify business intelligence ## Documentation topics Metabase's reference documentation. + ### Installation - [Installation overview](./installation-and-operation/start.md) @@ -61,6 +62,7 @@ Metabase's reference documentation. #### Query builder - [Asking questions](./questions/query-builder/introduction.md) +- [Visualizing data](./questions/sharing/visualizing-results.md) - [Custom expressions](./questions/query-builder/expressions.md) - [List of expressions](./questions/query-builder/expressions-list.md) - [Joining data](./questions/query-builder/join.md) @@ -68,7 +70,6 @@ Metabase's reference documentation. #### SQL and native queries - [The SQL editor](./questions/native-editor/writing-sql.md) -- [Data reference](./questions/native-editor/data-model-reference.md) - [SQL parameters](./questions/native-editor/sql-parameters.md) - [Referencing models and saved questions](./questions/native-editor/referencing-saved-questions-in-queries.md) - [SQL snippets](./questions/native-editor/sql-snippets.md) @@ -77,9 +78,8 @@ Metabase's reference documentation. #### Sharing - [Sharing answers](./questions/sharing/answers.md) -- [Visualizing data](./questions/sharing/visualizing-results.md) - [Setting and getting alerts](./questions/sharing/alerts.md) -- [Public links](./questions/sharing/public-links.md) +- [Public sharing](./questions/sharing/public-links.md) ### Dashboards @@ -104,7 +104,10 @@ Metabase's reference documentation. - [Organization overview](./exploration-and-organization/start.md) - [Basic exploration](./exploration-and-organization/exploration.md) - [Collections](./exploration-and-organization/collections.md) +- [History](./exploration-and-organization/history.md) +- [Data reference](./exploration-and-organization/data-model-reference.md) - [Events and timelines](./exploration-and-organization/events-and-timelines.md) +- [X-rays](./exploration-and-organization/x-rays.md) ### People @@ -139,9 +142,9 @@ Metabase's reference documentation. - [Embedding overview](./embedding/start.md) - [Embedding introduction](./embedding/introduction.md) -- [Signed embedding](./embedding/signed-embedding.md) - [Full-app embedding](./embedding/full-app-embedding.md) -- [Embedding example apps](https://github.com/metabase/embedding-reference-apps) +- [Signed embedding](./embedding/signed-embedding.md) +- [Parameters for signed embeds](./embedding/signed-embedding-parameters.md) ### Configuration @@ -151,7 +154,8 @@ Metabase's reference documentation. - [Email](./configuring-metabase/email.md) - [Slack](./configuring-metabase/slack.md) - [Environment variables](./configuring-metabase/environment-variables.md) -- [Metabase logs](./configuring-metabase/log-configuration.md) +- [Configuration file](./configuring-metabase/config-file.md) +- [Metabase log configuration](./configuring-metabase/log-configuration.md) - [Timezones](./configuring-metabase/timezones.md) - [Languages and localization](./configuring-metabase/localization.md) - [Appearance](./configuring-metabase/appearance.md) diff --git a/docs/api-documentation.md b/docs/api-documentation.md index 23ed152f02c0..28572333d70e 100644 --- a/docs/api-documentation.md +++ b/docs/api-documentation.md @@ -40,7 +40,6 @@ _* indicates endpoints used for features available on [paid plans](https://www.m - [Dataset](api/dataset.md) - [Email](api/email.md) - [Embed](api/embed.md) -- [Emitter](api/emitter.md) - [Field](api/field.md) - [GeoJSON](api/geojson.md) - [Google](api/google.md) diff --git a/docs/api/action.md b/docs/api/action.md index f01dd35dc4d6..d2a57a489792 100644 --- a/docs/api/action.md +++ b/docs/api/action.md @@ -53,28 +53,6 @@ Create a new HTTP action. * **`action`** -## `POST /api/action/:action-namespace/:action-name` - -Generic API endpoint for executing any sort of Action. - -### PARAMS: - -* **`action-namespace`** - -* **`action-name`** - -## `POST /api/action/:action-namespace/:action-name/:table-id` - -Generic API endpoint for executing any sort of Action with source Table ID specified as part of the route. - -### PARAMS: - -* **`action-namespace`** - -* **`action-name`** - -* **`table-id`** - ## `PUT /api/action/:id` ### PARAMS: diff --git a/docs/api/app.md b/docs/api/app.md index 7014d70bb0fd..803465fe92fa 100644 --- a/docs/api/app.md +++ b/docs/api/app.md @@ -27,18 +27,6 @@ Fetch a specific App. * **`id`** -## `GET /api/app/global-graph` - -Fetch the global graph of all App Permissions. - -You must be a superuser to do this. - -## `GET /api/app/graph` - -Fetch the graph of all App Permissions. - -You must be a superuser to do this. - ## `POST /api/app/` Endpoint to create an app. @@ -97,26 +85,6 @@ Endpoint to change an app. * **`nav_items`** value may be nil, or if non-nil, value must be an array. Each value may be nil, or if non-nil, value must be a map. -## `PUT /api/app/global-graph` - -Do a batch update of the global App Permissions by passing in a modified graph. - -You must be a superuser to do this. - -### PARAMS: - -* **`body`** value must be a map. - -## `PUT /api/app/graph` - -Do a batch update of the advanced App Permissions by passing in a modified graph. - -You must be a superuser to do this. - -### PARAMS: - -* **`body`** value must be a map. - --- [<< Back to API index](../api-documentation.md) \ No newline at end of file diff --git a/docs/api/card.md b/docs/api/card.md index 8cd8918dd792..539ffaefe7e4 100644 --- a/docs/api/card.md +++ b/docs/api/card.md @@ -28,7 +28,7 @@ Delete a Card. (DEPRECATED -- don't delete a Card anymore -- archive it instead. Get all the Cards. Option filter param `f` can be used to change the set of Cards that are returned; default is `all`, but other options include `mine`, `bookmarked`, `database`, `table`, `recent`, `popular`, and `archived`. See - corresponding implementation functions above for the specific behavior of each filter option. :card_index:. + corresponding implementation functions above for the specific behavior of each filter option. :card_index. ### PARAMS: diff --git a/docs/api/dashboard.md b/docs/api/dashboard.md index 9e464c1a373f..fdb67647f1d3 100644 --- a/docs/api/dashboard.md +++ b/docs/api/dashboard.md @@ -265,6 +265,8 @@ Copy a Dashboard. * **`collection_position`** value may be nil, or if non-nil, value must be an integer greater than zero. +* **`is_deep_copy`** value may be nil, or if non-nil, value must be a boolean. + * **`_dashboard`** ## `POST /api/dashboard/:id/cards` diff --git a/docs/api/emitter.md b/docs/api/emitter.md deleted file mode 100644 index bd4c7bdb2734..000000000000 --- a/docs/api/emitter.md +++ /dev/null @@ -1,57 +0,0 @@ ---- -title: "Emitter" -summary: | - API endpoints for Emitter. ---- - -# Emitter - -API endpoints for Emitter. - -## `DELETE /api/emitter/:emitter-id` - -Endpoint to delete an emitter. - -### PARAMS: - -* **`emitter-id`** - -## `POST /api/emitter/` - -Endpoint to create an emitter. - -### PARAMS: - -* **`action_id`** value must be an integer greater than zero. - -* **`card_id`** value may be nil, or if non-nil, value must be an integer greater than or equal to zero. - -* **`dashboard_id`** value may be nil, or if non-nil, value must be an integer greater than or equal to zero. - -* **`options`** value may be nil, or if non-nil, value must be a map. - -* **`parameter_mappings`** value may be nil, or if non-nil, value must be a map. - -## `POST /api/emitter/:id/execute` - -Execute a custom emitter. - -### PARAMS: - -* **`id`** - -* **`parameters`** value may be nil, or if non-nil, map of parameter name or ID -> map of parameter `:value` and `:type` of the value - -## `PUT /api/emitter/:emitter-id` - -Endpoint to update an emitter. - -### PARAMS: - -* **`emitter-id`** - -* **`emitter`** - ---- - -[<< Back to API index](../api-documentation.md) \ No newline at end of file diff --git a/docs/api/google.md b/docs/api/google.md index 9299b08147cb..bc3b19f5e60c 100644 --- a/docs/api/google.md +++ b/docs/api/google.md @@ -10,7 +10,7 @@ summary: | ## `PUT /api/google/settings` -Update Google Sign-In related settings. You must be a superuser to do this. +Update Google Sign-In related settings. You must be a superuser or have `setting` permission to do this. ### PARAMS: diff --git a/docs/api/ldap.md b/docs/api/ldap.md index 79ba83d46d5e..7c531a4e0c09 100644 --- a/docs/api/ldap.md +++ b/docs/api/ldap.md @@ -10,7 +10,7 @@ summary: | ## `PUT /api/ldap/settings` -Update LDAP related settings. You must be a superuser to do this. +Update LDAP related settings. You must be a superuser or have `setting` permission to do this. ### PARAMS: diff --git a/docs/api/permissions.md b/docs/api/permissions.md index 1fde64ed13de..d2bc6ea9f0df 100644 --- a/docs/api/permissions.md +++ b/docs/api/permissions.md @@ -24,12 +24,6 @@ Remove a User from a PermissionsGroup (delete their membership). * **`id`** -## `GET /api/permissions/execution/graph` - -Fetch a graph of execution permissions. - -You must be a superuser to do this. - ## `GET /api/permissions/graph` Fetch a graph of all Permissions. @@ -82,21 +76,6 @@ Add a `User` to a `PermissionsGroup`. Returns updated list of members belonging * **`is_group_manager`** value may be nil, or if non-nil, value must be a boolean. -## `PUT /api/permissions/execution/graph` - -Do a batch update of execution permissions by passing in a modified graph. The modified graph of the same - form as returned by the corresponding GET endpoint. - - Revisions to the permissions graph are tracked. If you fetch the permissions graph and some other third-party - modifies it before you can submit you revisions, the endpoint will instead make no changes and return a - 409 (Conflict) response. In this case, you should fetch the updated graph and make desired changes to that. - -You must be a superuser to do this. - -### PARAMS: - -* **`body`** value must be a map. - ## `PUT /api/permissions/graph` Do a batch update of Permissions by passing in a modified graph. This should return the same graph, in the same diff --git a/docs/configuring-metabase/appearance.md b/docs/configuring-metabase/appearance.md index ac30f02a87dd..f48c4a53df34 100644 --- a/docs/configuring-metabase/appearance.md +++ b/docs/configuring-metabase/appearance.md @@ -13,7 +13,9 @@ redirect_from: {% include plans-blockquote.html feature="Custom appearance" %} -Appearance setttings give you the option to white label Metabase to match your company’s branding. +Appearance settings give you the option to white label Metabase to match your company’s branding. + +If you're looking for date, time, number, or currency formatting, see [Formatting defaults](../data-modeling/formatting.md). ## Changing Metabase's appearance diff --git a/docs/configuring-metabase/config-file.md b/docs/configuring-metabase/config-file.md new file mode 100644 index 000000000000..cb8ffe125ec2 --- /dev/null +++ b/docs/configuring-metabase/config-file.md @@ -0,0 +1,125 @@ +--- +title: "Configuration file" +--- + +# Configuration file + +{% include plans-blockquote.html feature="Loading from a configuration file" self-hosted-only="true" %} + +On some paid, self-hosted plans, Metabase supports initialization on launch from a config file named `config.yml`. This file should either be in the current directory (the directory where the running Metabase JAR is located), or in a path specified by the `MB_CONFIG_FILE_PATH` [environment variable](./environment-variables.md). In this config file, you can specify: + +- Settings +- Users +- Databases + +The settings as defined in the config file work the same as if you set these settings in the Admin Settings in your Metabase. Settings defined in this configuration file will update any existing settings. If, for example, a database already exists (that is, you'd already added it via initial set up or **Admin settings** > **Databases**, Metabase will update the database entry based on the data in the config file). + +The config file settings are not treated as a hardcoded source of truth like [environment variables](./environment-variables.md) are. + +## Example `config.yml` file + +This file sets up a user account and two database connections. + +``` +version: 1 +config: + users: + - first_name: Cam + last_name: Era + password: 2cans3cans4cans + email: cam@example.com + databases: + - name: test-data (Postgres) + engine: postgres + details: + host: localhost + port: 5432 + password: {% raw %} "{{ env POSTGRES_TEST_DATA_PASSWORD }}" {% endraw %} + dbname: test-data + - name: Sample Dataset (Copy) + engine: h2 + details: + db: "/home/cam/metabase/resources/sample-database.db;USER=GUEST;PASSWORD={{env SAMPLE_DATASET_PASSWORD}}" +``` + +To determine which keys you can specify, simply look at the fields available in Metabase itself. + +## Referring to environment variables in the `config.yml` + +Environment variables can be specified with `{% raw %}{{ template-tags }}{% endraw %}` like `{% raw %}{{ env POSTGRES_TEST_DATA_PASSWORD }}{% endraw %}` or `{% raw %}[[options {{template-tags}}]]{% endraw %}`. + +Metabase doesn't support recursive expansion, so if one of your environment variables references _another_ environment variable, you're going to have a bad time. + +## Disable initial database sync + +When loading a data model from a serialized dump, you want to disable the scheduler so that the Metabase doesn't try to sync. + +To disable the initial database sync, you can add `config-from-file-sync-database` to the `settings` list and set the value to `false`. The setting `config-from-file-sync-database` must come _before_ the databases list, like so: + +``` +version: 1 +config: + settings: + config-from-file-sync-databases: false + databases: + - name: my-database + engine: h2 + details: ... +``` + +## List of settings + +In general, the settings you can set in the `settings` section of this config file map to the [environment variables](./environment-variables.md). + +The actual key that you include in the config file differs slightly from the format used for environment variables. For environment variables, the form is in screaming snake case, prefixed by an `MB`: + +``` +MB_NAME_OF_VARIABLE +``` + +Whereas in the config file, you'd translate that to: + +``` +name-of-variable +``` + +So for example, if you wanted to specify the `MB_EMAIL_FROM_NAME` not in an environment variable, but instead in the `config.yml` file: + +``` +version: 1 +config: + settings: + config-from-file-sync-databases: false + email-from-name: Stampy von Mails-a-lot + databases: + - name: my-database + engine: h2 + details: ... +``` + +## Users and Admins + +The first user created in a Metabase instance is an Admin. The first user listed in the config file may be designated an admin, but not necessarily, for example if someone has already spun up and logged into that Metabase for the first time, that user would be the automatically designated admin. Additionally, you can specify a user account as an admin by using the `is_superuser: true` key. + +In the following example: +``` +version: 1 +config: + users: + - first_name: a + last_name: b + password: metabot1 + email: a@b.com + - first_name: b + last_name: a + password: metabot1 + email: b@a.com + - first_name: c + last_name: a + password: metabot1 + is_superuser: true + email: c@a.com +``` +both users a@b.com and c@a.com will be admins: a@b.com because it's the first one on the list (if the instance is not yet configured) and c@a.com because it has the is_superuser flag. If an instance has already been configured, then a@b.com will be loaded as a normal user. + + diff --git a/docs/configuring-metabase/email.md b/docs/configuring-metabase/email.md index c4734faf3a88..0ceb07d9b46a 100644 --- a/docs/configuring-metabase/email.md +++ b/docs/configuring-metabase/email.md @@ -27,6 +27,8 @@ You should see this form: ![Email Credentials](images/EmailCredentials.png) +You might also want to specify [approved domains for notifications](./settings.md#approved-domains-for-notifications). + ## Google Apps 1. In the **SMTP host** field, enter smtp.gmail.com @@ -59,4 +61,4 @@ Check if [email quotas](https://docs.aws.amazon.com/ses/latest/dg/quotas.html) a ## Recommended settings * SSL is strongly recommended because it’s more secure and gives your account extra protection from threats. -* If your email service has a whitelist of email addresses that are allowed to send email, be sure to whitelist the email address that you put in the **From Address** field to ensure you and your teammates receive all emails from Metabase. \ No newline at end of file +* If your email service has a whitelist of email addresses that are allowed to send email, be sure to whitelist the email address that you put in the **From Address** field to ensure you and your teammates receive all emails from Metabase. diff --git a/docs/configuring-metabase/environment-variables.md b/docs/configuring-metabase/environment-variables.md index 250dce97d11f..fe85187fc9dd 100644 --- a/docs/configuring-metabase/environment-variables.md +++ b/docs/configuring-metabase/environment-variables.md @@ -77,7 +77,32 @@ Only available in [some plans](https://www.metabase.com/pricing)
Type: string
Default: `"{}"` -JSON object of primary colors used in charts and throughout Metabase. +JSON object of primary colors used in charts and throughout Metabase. Examples: + +To change the user interface colors: + +``` +{ + "brand":"#ff003b", + "filter":"#FF003B", + "summarize":"#FF003B" +} +``` + +To change the chart colors: + +``` +{ + "accent0":"#FF0005", + "accent1":"#E6C367", + "accent2":"#B9E68A", + "accent3":"#8AE69F", + "accent4":"#8AE6E4", + "accent5":"#8AA2E6", + "accent6":"#B68AE6", + "accent7":"#E68AD0" +} +``` ### `MB_APPLICATION_DB_MAX_CONNECTION_POOL_SIZE` @@ -188,6 +213,13 @@ Default: `true` Color log lines. When set to `false` it will disable log line colors. This is disabled on Windows. Related to [MB_EMOJI_IN_LOGS](#mb_emoji_in_logs). +### `MB_CONFIG_FILE_PATH` + +Type: string
+Default: `config.yml` + +This feature requires the `advanced-config` feature flag on your token. + ### `MB_CUSTOM_FORMATTING` Type: string
diff --git a/docs/configuring-metabase/localization.md b/docs/configuring-metabase/localization.md index 606a43d0fc04..c89f37652066 100644 --- a/docs/configuring-metabase/localization.md +++ b/docs/configuring-metabase/localization.md @@ -6,13 +6,20 @@ redirect_from: # Languages and localization +The **Localization** settings allow you to set global defaults for your Metabase instance. You can find **Localization** under **Admin settings** > **Settings**. + +## Default language + +Here you can set the default language (also called the "instance language") across your Metabase UI, system [emails](./email.md), [dashboard subscriptions](../dashboards/subscriptions.md), and [alerts](../questions/sharing/alerts.md). People can pick a different language from their own [account settings](../people-and-groups/account-settings.md). + ## Supported languages -Thanks to our amazing user community, Metabase has been translated into many different languages. Due to [the way we collect translations](#policy-for-adding-and-removing-translations), languages may be added or removed during major releases depending on translation coverage. +Thanks to our amazing user community, Metabase has been translated into many different languages. Due to [the way we collect translations](#translations), languages may be added or removed during major releases depending on translation coverage. The languages you can currently pick from are: - English (default) +- Arabic - Bulgarian - Catalan - Chinese (simplified) @@ -37,52 +44,60 @@ The languages you can currently pick from are: - Ukrainian - Vietnamese -## Policy for adding and removing translations +## Translations -Our community contributes to Metabase translations on our [POEditor project][metabase-poe]. If you'd like to help make Metabase available in a language you're fluent in, we'd love your help! +Our community contributes to Metabase translations on our [POEditor project](https://poeditor.com/join/project/ynjQmwSsGh). If you'd like to help make Metabase available in a language you're fluent in, we'd love your help! -For a new translation to be added to Metabase, it must reach 100%. Once it does, we add it in the next major or minor release of Metabase. All _existing_ translations in Metabase _must stay at 100%_ to continue being included in the next _major_ version of Metabase. This rule ensures that no one encounters a confusing mishmash of English and another language when using Metabase. +For a new language to be added to Metabase, it must reach 100%. Once it does, we add it in the next major or minor release of Metabase. All _existing_ languages in Metabase _must stay at 100%_ to continue being included in the next _major_ version of Metabase. This rule ensures that no one encounters a confusing mishmash of English and another language when using Metabase. We understand that this is a high bar, so we commit to making sure that before each major release, any additions or changes to text in the product are completed at least 10 calendar days before the release ships, at which point we notify all translators that a new release will be happening soon. Note that while we only remove languages in major releases, we are happy to add them back for minor releases, so it's always a good time to jump in and start translating. -[metabase-poe]: https://poeditor.com/join/project/ynjQmwSsGh +## Report timezone -## Localization +Use **report timezone** to set a default display time zone for dates and times in Metabase. The report timezone setting is a display setting only, so changing the report timezone won't affect the time zone of any data in your database. -The **Localization** settings allow you to set global defaults for your Metabase instance. Localization settings include options for: +Report timezone doesn't apply to `timestamp without time zone` data types, including the output of [`convertTimezone`](../questions/query-builder/expressions/converttimezone.md) expressions. For example: -- **Language** -- **Date and time** -- **Numbers** -- **Currency** +| Raw timestamp in your database | Data type | Report time zone | Displayed as | +| ---------------------------------------- | ----------------------------- | ---------------- | ---------------------- | +| `2022-12-28T12:00:00 AT TIME ZONE 'CST'` | `timestamp with time zone` | 'Canada/Eastern' | Dec 28, 2022, 7:00 AM | +| `2022-12-28T12:00:00-06:00` | `timestamp with offset` | 'Canada/Eastern' | Dec 28, 2022, 7:00 AM | +| `2022-12-28T12:00:00` | `timestamp without time zone` | 'Canada/Eastern' | Dec 28, 2022, 12:00 AM | -The **Localization** settings can be found in the **Admin Panel** under the **Settings** tab. +Report timezone is only supported for the following databases: + - BigQuery + - Druid + - MySQL + - Oracle + - PostgreSQL + - Presto + - Vertica -### Instance language +## First day of the week -The default language for all users across the Metabase UI, system emails, pulses, and alerts. Users can pick a different language from their own account settings page. +If you need to, you can change the first day of the week for your instance (the default is Sunday). Setting the first day of the week affects things like grouping by week and filtering in questions built using the [query builder](../questions/query-builder/introduction.md). This setting doesn't affect [SQL queries](../questions/native-editor/writing-sql.md). -### First day of the week +## Localization options -If you need to, you can change the first day of the week for your instance (the default is Sunday). Setting the first day of the week affects things like grouping by week and filtering in questions built using the [query builder](../questions/query-builder/introduction.md). This setting doesn't affect [SQL queries](../questions/native-editor/writing-sql.md). +**Localization options** allow you to set global default display formats for dates, times, numbers, and currencies. -### Localization options +You can also override these localization options for specific fields or questions. For more info, see [Formatting](../data-modeling/formatting.md). -**Dates and Times** +### Dates and times -- `Date style:` the way dates should be displayed in tables, axis labels, and tooltips. -- `Date separators:` you can choose between slashes, dashes, and dots here. -- `Abbreviate names of days and months:` whenever a date is displayed with the day of the week and/or the month written out, turning this setting on will display e.g. `January` as `Jan` or `Monday` as `Mon`. -- `Time style:` this lets you choose between a 12-hour or 24-hour clock to display the time by default where applicable. +- **Date style:** the way dates should be displayed in tables, axis labels, and tooltips. +- **Date separators:** you can choose between slashes (`2022/12/14`), dashes (`2022-12-14`), and dots (`2022.12.14.`). +- **Abbreviate names of days and months:** whenever a date is displayed with the day of the week and/or the month written out, turning this setting on will display e.g. "January" as "Jan" or "Monday" as "Mon". +- **Time style:** choose to display the time using either a 12 or 24-hour clock (e.g., 3:00 PM or 15:00). -**Numbers** +### Numbers -- `Separator style:` some folks use commas to separate thousands places, and others use periods. Here's where you can indicate which camp you belong to. +- **Separator style:** some people use commas to separate thousands places, and others use periods. Here's where you can indicate which camp you belong to. -**Currency** +### Currency -- `Unit of currency:` if you do most of your business in a particular currency, you can specify that here. -- `Currency label style:` whether you want to have your currencies labeled with a symbol, a code (like `USD`), or its full name. -- `Where to display the unit of currency:` this pertains specifically to tables, and lets you choose if you want the currency labels to appear only in the column heading, or next to each value in the column. +- **Unit of currency:** if you do most of your business in a particular currency, you can specify that here. +- **Currency label style:** whether you want to have your currencies labeled with a symbol, a code (like "USD"), or its full name. +- **Where to display the unit of currency:** this pertains specifically to tables, and lets you choose if you want the currency labels to appear only in the column heading, or next to each value in the column. diff --git a/docs/configuring-metabase/settings.md b/docs/configuring-metabase/settings.md index c07dcc57ea3c..db11f64194ef 100644 --- a/docs/configuring-metabase/settings.md +++ b/docs/configuring-metabase/settings.md @@ -4,54 +4,52 @@ redirect_from: - /docs/latest/administration-guide/08-configuration-settings --- -## General settings +# General settings This section contains settings for your whole instance, like its URL, the reporting timezone, and toggles for disabling or enabling some of Metabase's optional features. -You can configure these settings in the **General** section of the **Settings** tab in the **Admin Panel**. +You can configure these settings from **Settings** > **Admin Settings** > **General**. -### Site Name +## Site name How you’d like to refer to this instance of Metabase. -### Site URL +## Site URL -The base URL of this Metabase instance. The base URL is used in emails to allow users to click through to their specific instance. Make sure to include http:// or https:// to make sure it’s reachable. +The site URL is the web address that people use to access your Metabase instance. Make sure to include `http://` or `https://` to make sure it’s reachable. ### Redirect to HTTPS -Force all traffic to use HTTPS via redirect, if the site can serve over HTTPS. +By default, Metabase is served over HTTP. -_Default: disabled_. +To force all traffic to use HTTPS via redirect, click `http://` (under the **Site URL** section) and select `https://` from the dropdown menu. -For example, if you are serving your Metabase application at "example.com", and you enable HTTPS redirect, when a user enters an address like "example.com/data" in their browser's address bar, the user will be automatically redirected to a secure connection at `https://example.com/data`. +For example, say you enable HTTPS redirect for a Metabase instance at the site URL "example.com". When someone enters an address like `example.com/data` in their browser's address bar, they'll get automatically redirected to a secure connection at `https://example.com/data`. > Note: if you haven't set up HTTPS on your server, Metabase will not let you enable HTTPS redirect. Instead, you'll get a warning saying "It looks like HTTPS is not properly configured." -### Email Address for Help Requests +## Email address for help requests This email address will be displayed in various messages throughout Metabase when users encounter a scenario where they need assistance from an admin, such as a password reset request. -### Report Timezone +## Approved domains for notifications -The **report timezone** sets the default time zone for displaying times. The timezone is used when breaking out data by dates. +Allowed email address domain(s) for new [dashboard subscriptions](../dashboards/subscriptions.md) and [alerts](../questions/sharing/alerts.md). To specify multiple domains, separate each domain with a comma, with no space in between (e.g., "domain1,domain2"). To allow all domains, leave the field empty. This setting doesn't affect existing subscriptions. -_Setting the default timezone will not change the timezone of any data in your database_. If the underlying times in your database aren't assigned to a timezone, Metabase will use the report timezone as the default timezone. +## Anonymous tracking -### Anonymous Tracking +This option turns determines whether or not you allow [anonymous data about your usage of Metabase](../installation-and-operation/information-collection.md) to be sent back to us to help us improve the product. [Your database’s data is never tracked or sent](https://www.metabase.com/security). -This option turns determines whether or not you allow [anonymous data about your usage of Metabase](../installation-and-operation/information-collection.md) to be sent back to us to help us improve the product. _Your database’s data is never tracked or sent_. +## Friendly table and field names -### Enable X-rays +By default, Metabase attempts to make field and table names more readable by changing things like `somehorriblename` to `Some Horrible Name`. This does not work well for languages other than English, or for fields that have lots of abbreviations or codes in them. If you'd like to turn this setting off, you can do so from the Admin Panel under **Settings** > **Admin settings** > **General**. -[X-rays](../exploration-and-organization/x-rays.md) are a great way to allow your users to quickly explore your data or interesting parts of charts, or to see a comparison of different things. But if you're dealing with data sources where allowing users to run x-rays on them would incur burdonsome performance or monetary costs, you can turn them off here. +To manually label field or table names in Metabase, check out the [Data Model](../data-modeling/metadata-editing.md) section in your admin settings. -### Enabled Nested Queries +## Enable nested queries By default, Metabase allows your users to use a previously saved question as a source for queries. If you have a lot of slow running queries, you may want to switch this option off, as performance problem can occur. -### Friendly Table and Field Names +## Enable X-rays -By default, Metabase attempts to make field and table names more readable by changing things like `somehorriblename` to `Some Horrible Name`. This does not work well for languages other than English, or for fields that have lots of abbreviations or codes in them. If you'd like to turn this setting off, you can do so from the Admin Panel under Settings > General > Friendly Table and Field Names. - -To manually fix field or table names if they still look wrong, you can go to the Metadata section of the Admin Panel, select the database that contains the table or field you want to edit, select the table, and then edit the name(s) in the input boxes that appear. +[X-rays](../exploration-and-organization/x-rays.md) are a great way for people to get quick summary stats on your data. If these X-ray queries get too slow or expensive, you can turn them off here. diff --git a/docs/configuring-metabase/start.md b/docs/configuring-metabase/start.md index 7d1ad1109d32..3623a49dbe0e 100644 --- a/docs/configuring-metabase/start.md +++ b/docs/configuring-metabase/start.md @@ -20,7 +20,23 @@ Set up email for [Alerts](../questions/sharing/alerts.md) and [Dashboard subscri Set up Slack for Alerts and Dashboard subscriptions. -## [Localization](./localization.md) +## [Environment variables](./environment-variables.md) + +Configure Metabase on launch via environment variables. + +## [Configuration file](./config-file.md) + +On self-hosted paid plans, you can configure Metabase via a configuration file. + +## [Metabase logs configuration](./log-configuration.md) + +Tell Metabase what to log. + +## [Timezones](./timezones.md) + +Guidance on timezone settings. + +## [Languages and localization](./localization.md) Set language, datetime, and currency settings. @@ -28,10 +44,14 @@ Set language, datetime, and currency settings. Customize colors, fonts, and other visual elements. -## [Caching](./caching.md) +## [Caching query results](./caching.md) Cache query results for faster loading times. -## [Metabase logs configuration](./log-configuration.md) +## [Custom maps](./custom-maps.md) -Tell Metabase what to log. +Upload custom maps to your Metabase. + +## [Customizing the Metabase Jetty webserver](./customizing-jetty-webserver.md) + +Set SSL and port settings for the Jetty webserver. diff --git a/docs/configuring-metabase/timezones.md b/docs/configuring-metabase/timezones.md index 859394f5c4ab..31081adf9646 100644 --- a/docs/configuring-metabase/timezones.md +++ b/docs/configuring-metabase/timezones.md @@ -8,6 +8,8 @@ redirect_from: Metabase does its best to ensure proper and accurate reporting in whatever timezone you desire, but timezones are a complicated beast so it's important to abide by some recommendations listed below to ensure your reports come out as intended. +## Time zone settings + The following places where timezones are set can all impact the data you see: - `Database` - includes global database timezone settings, specific column type settings, and even individual data values. @@ -15,14 +17,31 @@ The following places where timezones are set can all impact the data you see: - `Metabase` - inside Metabase the reporting timezone setting (if set) will influence how your data is reported. - `Metabase Cloud` - if you need to change the timezone on the server that hosts your Metabase Cloud instance, please [contact support](https://www.metabase.com/help-premium). +## Recommended settings + To ensure proper reporting it's important that timezones be set consistently in all places. Metabase recommends the following settings: -- Make sure all of your database columns are properly setup to include timezone awareness. -- Unless you have a special need it's best to set your database reporting timezone to UTC and store all of your date/time related values in UTC. +- Make sure all of your database columns are properly setup to include [time zone awareness](#data-types). +- Unless you have a special need it's best to set your database reporting time zone to UTC and store all of your date/time related values in UTC. - Configure your JVM to use the same timezone you want to use for reporting, which ideally should also match the timezone of your database. - Set the Metabase `Report Timezone` to match the timezone you want to see your reports in, again, this should match the rest of the timezone settings you've made. -Common Pitfalls: +## Data types + +You can make your database columns time zone aware by storing them as specific data types, such as: + +| Data type | Description | Example | +| ----------------------------- | ----------------------------------------- | ---------------------------------------------------- | +| `timestamp with time zone` | Knows about location. | `2022-12-28T12:00:00 AT TIME ZONE 'America/Toronto'` | +| `timestamp with offset` | Knows about the time difference from UTC. | `2022-12-28T12:00:00-04:00` | +| `timestamp without time zone` | No time zone info. | `2022-12-28T12:00:00` | + +The exact data type will depend on your database. Some Metabase features only work with specific data types: + +- [Report timezone setting](../configuring-metabase/localization.md#report-timezone) +- [`converttimezone` custom expression](../questions/query-builder/expressions/converttimezone.md) + +## Common pitfalls 1. Your database is using date/time columns without any timezone information. Typically when this happens your database will assume all the data is from whatever timezone the database is configured in or possible just default to UTC (check your database vendor to be sure). 2. Your JVM timezone is different from your Metabase `Report Timezone` choice. This is a very common issue and can be corrected by launching java with the `-Duser.timezone=` option properly set to match your Metabase report timezone. diff --git a/docs/dashboards/filters.md b/docs/dashboards/filters.md index 7aada0f79aa5..61fe97239518 100644 --- a/docs/dashboards/filters.md +++ b/docs/dashboards/filters.md @@ -94,9 +94,9 @@ Once you’ve added a filter to your dashboard, just click on the filter to sele You can also set up a dashboard question to [update a filter on click](./interactive.md#use-a-chart-to-filter-a-dashboard). -## Choosing between a dropdown or autocomplete for your filter +## Default filter types -If the column you're using for a filter has more than 100 unique values, you'll now automatically see a search box with autocomplete suggestions: +If the column you're using for a filter has more than 100 unique values, you'll automatically see a search box with autocomplete suggestions. ![Autocomplete](./images/autocomplete.png) @@ -108,9 +108,9 @@ In both cases, you can pick one or multiple selections for your filter. ![Multi-select](./images/multi-select.png) -If Metabase somehow picked the wrong behavior for your field, admins can go to the [Data Model](../data-modeling/metadata-editing.md) section of the admin panel and click on the **gear** icon by the field in question to specify a list, search box, or plain input box. +## Creating a dropdown filter -![Search options](./images/search-options.png) +To override the default text box or search box filter widget, ask your Metabase admin to help you [update the column's settings](../data-modeling/metadata-editing.md#changing-a-search-box-filter-to-a-dropdown-filter) on the Data Model page. ## Linking filters diff --git a/docs/dashboards/interactive.md b/docs/dashboards/interactive.md index 87883dd52573..80ae148e15c8 100644 --- a/docs/dashboards/interactive.md +++ b/docs/dashboards/interactive.md @@ -64,6 +64,8 @@ Possible destinations include: - Saved questions - URLs +Internal Metabase destinations (dashboards or saved questions) will load in the same browser tab or window. External URLs will open in a new tab or window. + ## Passing values to the destination If you're linking to a dashboard or a SQL question that has filters, you can pass values from the current dashboard to filters in the destination. diff --git a/docs/dashboards/introduction.md b/docs/dashboards/introduction.md index 8c26891841ad..1ea9e99bc507 100644 --- a/docs/dashboards/introduction.md +++ b/docs/dashboards/introduction.md @@ -24,7 +24,17 @@ In the top right of the screen, click the **+** icon to open the **Create** menu ![Create Dashboard](images/create.png) -If you don't want to build a dashboard from scratch, or want to experiment by making changes to an existing dashboard without affecting the original, you can **duplicate** an existing dashboard. From an existing dashboard, click on the **...** menu in the upper right, and select **Duplicate**. +## Duplicating a dashboard + +If you don't want to build a dashboard from scratch, or want to experiment by making changes to an existing dashboard without affecting the original, you can **duplicate** an existing dashboard. + +To duplicate a dashboard, click on the **...** menu in the upper right of the dashboard, and select **Duplicate**. + +By default, Metabase will create a new dashboard, with copies of the questions in the original dashboard, and save everything to the collection you specify. + +If you don't wish to copy the dashboard's underlying questions, check the box that says **Only duplicate the dashboard**. Metabase will copy the dashboard and refer to the original questions. + +In both cases, duplication only includes the dashboard, its card layout, filters, and (optionally) the questions. What's not copied: dashboard subscriptions, or any sharing or embedding data settings. For example, if you copy a dashboard that has been made public, that copied dashboard will not be public by default. ## Adding saved questions to a dashboard @@ -181,13 +191,11 @@ The part that says `refresh=60` sets the dashboard to automatically refresh ever There is one important limitation with the `fullscreen` option: for security reasons, many browsers require user interaction to initiate fullscreen. In those browsers, using the `fullscreen` option will enable the fullscreen UI in Metabase, but it won't expand the browser content to fill the screen. To ensure the dashboard occupies the entire screen, either activate fullscreen by clicking the button in the UI, or use the `fullscreen` URL option and launch the browser in fullscreen or kiosk mode. -## Archiving a dashboard - -Archiving a dashboard removes the dashboard from searches and collections. Archiving a dashboard does not archive the individual saved questions on it — it just archives the dashboard. +## Dashboard version history -To archive a dashboard, click the **pencil** icon to enter edit mode, then click the **...** menu, and select **Archive**. +For [questions](../questions/start.md), dashboards, and [models](../data-modeling/models.md), Metabase keeps a version history for the previous fifteen versions of that item. -To view all archived items, click the **menu** icon in the top-right of any collection page. You can **unarchive** a dashboard by clicking the icon of the box with the upward arrow next to that dashboard. +See [History](../exploration-and-organization/history.md). ## Tips on creating helpful dashboards diff --git a/docs/data-modeling/metadata-editing.md b/docs/data-modeling/metadata-editing.md index 1b9d6a671be9..2a1f6220fe51 100644 --- a/docs/data-modeling/metadata-editing.md +++ b/docs/data-modeling/metadata-editing.md @@ -141,6 +141,16 @@ You can manually change the user interface for the filter to: ![Filter options](./images/filter-options.png) +## Changing a search box filter to a dropdown filter + +1. Go to **Settings** > **Admin settings** > **Data Model**. +2. Select the database, schema, table, and field in question. +3. Click the **gear** icon to view all the field’s settings. +4. Set **Field Type** to “Category”. +5. Set **Filtering on this field** to “A list of all values". + +This setting will run a query against your database to get the first 1,000 distinct values (ordered ascending) for that field and cache the first 100kB of text to display in the dropdown menu. If you have columns with more than 1,000 distinct values, or columns with text-heavy data, we recommend setting **Filtering on this field** to "Search box" instead. For more info, see [How database scans work](../databases/connecting.md#how-database-scans-work). + ## Column order Metabase will default to the column order native to the database. diff --git a/docs/data-modeling/models.md b/docs/data-modeling/models.md index 03b16d03b225..d1aece6352b5 100644 --- a/docs/data-modeling/models.md +++ b/docs/data-modeling/models.md @@ -98,7 +98,7 @@ You can refer to a model in a SQL query just like you can refer to a saved quest ``` {% raw %} -SELECT * FROM {{#1}} +SELECT * FROM {{#1-customer-model}} {% endraw %} ``` @@ -106,15 +106,21 @@ Or as a [common table expression (CTE)][cte]: ``` {% raw %} -WITH model AS {{#3807}} +WITH model AS {{#3807-invoice-model}} SELECT * FROM model; {% endraw %} ``` -## Model history +Simply typing `{% raw %}{{#}} {% endraw %}` will allow you to search for models (for example, you could type in `{% raw %}{{#customer}}{% endraw %}` to search models, questions, and tables with the word "customer" in the title. -Just like with saved questions, you can click on the model name with the down arrow to bring up the model's sidebar, then click on **History** at the bottom to bring up a list of the changes made to the model over time, and by whom. +You can also use the data reference sidebar to browse the models available. To open the data reference sidebar, click on the **book** icon. + +## Model version history + +For [questions](../questions/start.md), [dashboards](../dashboards/start.md), and models, Metabase keeps a version history for the previous fifteen versions of that item. You can view changes, revert to previous versions, and archive outdated items. + +See [History](../exploration-and-organization/history.md). ## Verifying a model diff --git a/docs/databases/connecting.md b/docs/databases/connecting.md index 6a6cc72815f1..6e1792dace7b 100644 --- a/docs/databases/connecting.md +++ b/docs/databases/connecting.md @@ -2,6 +2,7 @@ title: Adding and managing databases redirect_from: - /docs/latest/administration-guide/01-managing-databases + - /docs/latest/databases/connections/sql-server --- # Adding and managing databases @@ -20,8 +21,7 @@ Although connection details differ database to database, in general you'll need The databases listed below have official drivers maintained by the Metabase team. Customers on [paid plans](https://www.metabase.com/pricing) will get official support. -If you don't see your database listed here, see [partner and community drivers](../developers-guide/partner-and-community-drivers.md#partner-drivers). - +- [Amazon Athena](./connections/athena.md) - [BigQuery](./connections/bigquery.md) (Google Cloud Platform) - Druid - [Google Analytics](./connections/google-analytics.md) @@ -34,10 +34,12 @@ If you don't see your database listed here, see [partner and community drivers]( - Redshift (Amazon Web Services) - [Snowflake](./connections/snowflake.md) - SparkSQL -- [SQL Server](./connections/sql-server.md) +- SQL Server - SQLite - [Vertica](./connections/vertica.md) +If you don't see your database listed here, see [partner and community drivers](../developers-guide/partner-and-community-drivers.md#partner-drivers). + ## Connecting to databases hosted by a cloud provider For provider-specific connection details, like connecting to a PostgreSQL data warehouse on RDS: @@ -105,19 +107,23 @@ A fingerprinting query examines the first 10,000 rows from each column and uses ## Syncing and scanning databases -Metabase runs sync and scan queries in order to show tables and columns, populate dropdown menus with the right values, and suggest helpful visualizations. Metabase does _not_ copy any data from your database---it only maintains lists of the tables and columns. +Metabase runs syncs and scans to stay up to date with your database. Syncs get updated schemas to display in the [Data Browser](https://www.metabase.com/learn/getting-started/data-browser). Scans take samples of column values to populate filter dropdown menus and suggest helpful visualizations. Metabase does not store _complete_ tables from your database. ### How database syncs work -A Metabase **sync** query gets a list of updated table names, column names, and column data types from your database. The query runs against your database during setup, and again every hour by default. It's very fast with most relational databases, but can be slower with MongoDB and some [community-built database drivers](../developers-guide/partner-and-community-drivers.md). Syncing can't be turned off completely, otherwise Metabase wouldn't work. +A Metabase **sync** is a query that gets a list of updated table and view names, column names, and column data types from your database. This query runs against your database during setup, and again every hour by default. This scanning query is fast with most relational databases, but can be slower with MongoDB and some [community-built database drivers](../developers-guide/partner-and-community-drivers.md). Syncing can't be turned off completely, otherwise Metabase wouldn't work. ### How database scans work -A Metabase **scan** query caches the column _values_ for filter dropdowns by looking at the first 1,000 distinct records from each table. A scan is more intensive than a sync query, so it only runs once during setup, and again once a day by default. If you [disable scans](#scheduling-database-scans) entirely, you'll need to bring things up to date by running [manual scans](#manually-scanning-column-values). +A Metabase **scan** is a query that caches the column _values_ for filter dropdowns by looking at the first 1,000 distinct records from each table, in ascending order. For each record, Metabase only stores the first 100 kilobytes of text, so if you have data with 1,000 characters each (like addresses), and your column has more than 200 unique addresses, Metabase will only cache the first 100 values from the scan query. + +Cached column values are displayed in filter dropdown menus. If people type in the filter search box for values that aren't in the first 1,000 distinct records or 100kB of text, Metabase will run a query against your database to look for those values on the fly. + +A scan is more intensive than a sync query, so it only runs once during setup, and again once a day by default. If you [disable scans](#scheduling-database-scans) entirely, you'll need to bring things up to date by running [manual scans](#manually-scanning-column-values). ### Getting tables, columns, and values for the first time -When Metabase first connects to your database, it performs a **scan** to determine the metadata of the columns in your tables and automatically assign each column a [semantic type](). +When Metabase first connects to your database, it performs a **scan** to determine the metadata of the columns in your tables and automatically assign each column a [semantic type](../data-modeling/field-types.md). During the scan, Metabase also takes a sample of each table to look for URLs, JSON, encoded strings, etc. You can map table and column metadata to new values from **Admin settings** > **Data model**. For more on editing metadata, check out [the Data Model page: editing metadata](../data-modeling/metadata-editing.md). @@ -154,6 +160,14 @@ To forget the data that Metabase has stored from previous [database scans](#sync ![Re-scan options](./images/re-scan-options.png) +### Syncing and scanning using the API + +Metabase syncs and scans regularly, but if the database administrator has just changed the database schema, or if a lot of data is added automatically at specific times, you may want to write a script that uses the [Metabase API](https://www.metabase.com/learn/administration/metabase-api) to force a sync or scan. [Our API](../api-documentation.md) provides two ways to initiate a sync or scan of a database: + +1. Using a a session token: the `/api/database/:id/sync_schema` or `api/database/:id/rescan_values` endpoints. These endpoints do the same things as going to the database in the Admin Panel and choosing **Sync database schema now** or **Re-scan field values now** respectively. To use these endpoints, you have to authenticate with a user ID and pass a session token in the header of your request. + +2. Using an API key: `/api/notify/db/:id`. We created this endpoint so that people could notify their Metabase to sync after an [ETL operation](https://www.metabase.com/learn/analytics/etl-landscape) finishes. To use this endpoint, you must pass an API key by defining the `MB_API_KEY` environment variable. + ## Deleting databases **Caution: Deleting a database is irreversible! All saved questions and dashboard cards based on the database will be deleted as well!** diff --git a/docs/databases/connections/athena.md b/docs/databases/connections/athena.md new file mode 100644 index 000000000000..4c8a6e9584d0 --- /dev/null +++ b/docs/databases/connections/athena.md @@ -0,0 +1,79 @@ +--- +title: Amazon Athena +--- + +# Amazon Athena + +Connecting Metabase to Athena depends on where Metabase is running. + +## Connecting to Athena + +To connect Metabase to Athena, you'll need to input your IAM credentials: + +- Access key +- Secret Key + +Metabase will encrypt these credentials. + +If you use other AWS services, we recommend that you create a special AWS Service Account that only has the permissions required to run Athena, and input the IAM credentials from that account to connect Metabase to Athena. + +See [Identity and access management in Athena](https://docs.aws.amazon.com/athena/latest/ug/security-iam-athena.html). + +## Connecting using AWS Default Credentials Chain + +If you're running Metabase on AWS and want to use [AWS Default Credentials Chain](https://docs.aws.amazon.com/sdk-for-java/v1/developer-guide/credentials.html#credentials-default), leave the Access and Secret keys blank. + +- For EC2, you can use [instance profiles](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2_instance-profiles.html). +- For ECS, you can use [IAM roles for tasks](https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-iam-roles.html) + +In both cases, the Athena driver will automatically fetch session credentials based on which IAM role you've configured. + +## Connection settings + +### Display name + +How Metabase should refer to the data source in its user interface. + +### Region + +The region where your Athena database is hosted, e.g., `us-east-1`. + +### Workgroup + +For example: `primary`. See [documentation on workgroups](https://docs.aws.amazon.com/athena/latest/ug/user-created-workgroups.html) + +### S3 Staging directory + +This S3 staging directory must be in the same region you specify above. + +### Catalog + +You can use a different [catalog](https://docs.aws.amazon.com/athena/latest/ug/understanding-tables-databases-and-the-data-catalog.html) (for example if you're using federated queries). + +## Advanced options + +### Additional Athena connection string options + +You can append additional options to the connection string. For example, to disable result set streaming and enable TRACE-level debugging: + +``` +UseResultsetStreaming=0;LogLevel=6. +``` + +For more connection options, see Simba Athena JDBC Driver with SQL Connector's [Installation and Configuration Guide](https://s3.amazonaws.com/athena-downloads/drivers/JDBC/SimbaAthenaJDBC_2.0.13/docs/Simba+Athena+JDBC+Driver+Install+and+Configuration+Guide.pdf). + +### Rerun queries for simple explorations + +We execute the underlying query when you explore data using Summarize or Filter. This is on by default but you can turn it off if performance is slow. + +### Choose when syncs and scans happen + +By default, Metabase does a lightweight hourly sync and an intensive daily scan of field values. If you have a large database, turn this on to make changes. + +### Periodically refingerprint tables + +This enables Metabase to scan for additional field values during syncs allowing smarter behavior, like improved auto-binning on your bar charts. + +### Default result cache duration + +How long to keep question results. By default, Metabase will use the value you supply on the cache settings page, but if this database has other factors that influence the freshness of data, it could make sense to set a custom duration. You can also choose custom durations on individual questions or dashboards to help improve performance. diff --git a/docs/databases/connections/mongodb.md b/docs/databases/connections/mongodb.md index fd90a979f9bf..c9ff4a6e70e1 100644 --- a/docs/databases/connections/mongodb.md +++ b/docs/databases/connections/mongodb.md @@ -111,7 +111,7 @@ To make sure you are using the correct connection configuration: ## I added fields to my database but don't see them in Metabase -Metabase may not sync all of your fields, as it only scans the first 200 documents in a collection to get a sample of the fields the collection contains. Since any document in a MongoDB collection can contain any number of fields, the only way to get 100% coverage of all fields would be to scan every single document in every single collection, which would put too much strain on your database (so we don't do that). +Metabase may not sync all of your fields, as it only scans the first ten thousand documents in a collection to get a sample of the fields the collection contains. Since any document in a MongoDB collection can contain any number of fields, the only way to get 100% coverage of all fields would be to scan every single document in every single collection, which would put too much strain on your database (so we don't do that). One workaround is to include all possible keys in the first document of the collection, and give those keys null values. That way, Metabase will be able to recognize the correct schema for the entire collection. diff --git a/docs/databases/connections/sql-server.md b/docs/databases/connections/sql-server.md deleted file mode 100644 index 715df7bc3a7c..000000000000 --- a/docs/databases/connections/sql-server.md +++ /dev/null @@ -1,13 +0,0 @@ ---- -title: Microsoft SQL Server -redirect_from: - - /docs/latest/administration-guide/databases/sql-server ---- - -# Microsoft SQL Server - -Also known as MSSQL. - -## Connecting to SQL Server with dynamic ports - -If your SQL Server uses dynamic ports, simply leave the port field blank. diff --git a/docs/developers-guide/contributing.md b/docs/developers-guide/contributing.md index 8ab8f3d187f2..479475ef4b6c 100644 --- a/docs/developers-guide/contributing.md +++ b/docs/developers-guide/contributing.md @@ -93,7 +93,9 @@ By our definition, "Bugs" are situations where the program doesn't do what it wa ### Help with Documentation -There are a lot of docs. We often have difficulties keeping them up to date. If you are reading them and you notice inconsistencies, errors or outdated information, please help up keep them current! +There are a lot of docs, which means keeping them up to date is hard. If you notice inconsistencies, errors, or outdated information, please help us keep them current! + +Note that **we cannot accept translations for documentation at this time**. We support [in-app translations](../configuring-metabase/localization.md), and only support languages that have 100% coverage. But 1) the in-app text is orders of magnitude shorter than our docs, 2) it changes at a slower pace, and 3) we have a lot of people help out. We may consider supporting docs in multiple languages in the future, but for now we need to focus our resources on improving our existing documentation (and expanding it to include all of the new features we're adding). ### Working on features diff --git a/docs/developers-guide/e2e-tests.md b/docs/developers-guide/e2e-tests.md index 96f7cda7a208..8dee153d907b 100644 --- a/docs/developers-guide/e2e-tests.md +++ b/docs/developers-guide/e2e-tests.md @@ -46,7 +46,7 @@ Try to avoid repeatedly testing pieces of the application incidentally. For exam ## Cypress Documentation -* Introduction: https://docs.cypress.io/guides/core-concepts/introduction-to-cypress.html#Querying-by-Text-Content +* Introduction: https://docs.cypress.io/guides/core-concepts/introduction-to-cypress.html * Commands: https://docs.cypress.io/api/api/table-of-contents.html * Assertions: https://docs.cypress.io/guides/references/assertions.html diff --git a/docs/developers-guide/frontend.md b/docs/developers-guide/frontend.md index d96e20e553ae..3ce494d802fd 100644 --- a/docs/developers-guide/frontend.md +++ b/docs/developers-guide/frontend.md @@ -113,148 +113,6 @@ Any additional selectors and actions defined in the entities' `objectSelectors` You can also use the Redux actions and selectors directly, for example, `dispatch(Users.actions.loadList())` and `Users.selectors.getList(state)`. -## Forms - -Metabase includes a comprehensive custom React and [`redux-form`](https://redux-form.com/5.2.3/) based form library. It also integrates with Metabase's [Entities](https://github.com/metabase/metabase/wiki/Frontend:-Entity-Loaders) system. - -The core React component of the system is [`metabase/containers/Form`](https://github.com/metabase/metabase/blob/master/frontend/src/metabase/containers/Form.jsx). - -### Form Definitions - -Form definitions can be provided in two different ways, with a JavaScript-based form definition object, or inline React `` elements. - -Pass a form definition to the `form` prop: - -```javascript -
alert(JSON.stringify(values))} -/> -``` - -`fields` and `initial` (for initial values) can be provided directly or as functions that dynamically compute them based on the current form state and additional props. - -```javascript -{ - "fields": (values) => [ - { name: "a", type: } -``` - -`initial`, `normalize`, and `validate` properties can be provided at the top-level, or per-field. They can also be provided as props to the `` and `` components For definitions can be provided - -### Custom Layout - -Form definition can also be provided via `` React elements (exported from the same `metabase/containers/Form` module), which will also serve as the layout. - -```javascript -import Form, { FormField, FormFooter } from "metabase/containers/Form"; - - alert(JSON.stringify(values))}> - - - -; -``` - -You can also provide both the `form` prop and children `` elements, in which case the `form` prop will be merged with the ``s' props. - -### Custom Widgets - -Built-in field `type`s are defined in [metabase/components/form/FormWidget](https://github.com/metabase/metabase/blob/master/frontend/src/metabase/components/form/FormWidget.jsx#L17-L28). You can also provide a React component as the `type` property. - -### Validation - -You might have noticed the `validate` API above. These are simple chainable validators compatible with this form library, and are provided by [`metabase/lib/validate`](https://github.com/metabase/metabase/blob/master/frontend/src/metabase/lib/validate.js). You can add additional validators in that file. - -Server-side validation and other errors are returned in a standard format understood by `
`. - -Field-level errors: - -```json -{ "errors": { "field_name": "error message" } } -``` - -Top-level errors: - -```json -{ "message": "error message" } -``` - -### Integration with Entities - -The Form library is integrated with Metabase's [Entities](https://github.com/metabase/metabase/wiki/Frontend:-Entity-Loaders) system (via the [`EntityForm`](https://github.com/metabase/metabase/blob/master/frontend/src/metabase/entities/containers/EntityForm.jsx) component), so that every entity includes a `Form` component that can be used like so: - -```javascript - -``` - -which uses the default `form` defined on the entity, e.x. - -```javascript -const Users = createEntity({ - name: "users", - path: "/api/user", - - form: { - fields: [ - { name: "email" } - ] - } - - // Alternatively, it will take the first form from the `forms` object: - // form: { - // default: { - // fields: [ - // { name: "email" } - // ] - // } - // } -} -``` - -You can also explicitly pass a different form object: - -```javascript - -``` - -Entity `Form`s will automatically be wired up to the correct REST endpoints for creating or updating entities. - -If you need to load an object first, they compose nicely with the Entities `Loader` render prop: - -```javascript - - {({ user }) => } - -``` - -Or higher-order component: - -```javascript -Users.load({ id: (state, props) => props.params.userId })(Users.Form); -``` - ## Style Guide ### Set up Prettier diff --git a/docs/developers-guide/partner-and-community-drivers.md b/docs/developers-guide/partner-and-community-drivers.md index 88d84fb2170a..f80b60536fde 100644 --- a/docs/developers-guide/partner-and-community-drivers.md +++ b/docs/developers-guide/partner-and-community-drivers.md @@ -60,7 +60,6 @@ You install these drivers at your own risk. The plugins will run as part of your | Database | GitHub Stars | Last release (_if available_) | | -------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------- | -| [Amazon Athena](https://github.com/dacort/metabase-athena-driver) | ![GitHub stars](https://img.shields.io/github/stars/dacort/metabase-athena-driver) | ![GitHub (Pre-)Release Date](https://img.shields.io/github/release-date-pre/dacort/metabase-athena-driver) | | [ClickHouse](https://github.com/enqueue/metabase-clickhouse-driver) | ![GitHub stars](https://img.shields.io/github/stars/enqueue/metabase-clickhouse-driver) | ![GitHub (Pre-)Release Date](https://img.shields.io/github/release-date-pre/enqueue/metabase-clickhouse-driver) | | [CSV](https://github.com/Markenson/csv-metabase-driver) | ![GitHub stars](https://img.shields.io/github/stars/Markenson/csv-metabase-driver) | ![GitHub (Pre-)Release Date](https://img.shields.io/github/release-date-pre/Markenson/csv-metabase-driver) | | [Cube.js](https://github.com/lili-data/metabase-cubejs-driver) | ![GitHub stars](https://img.shields.io/github/stars/lili-data/metabase-cubejs-driver) | ![GitHub (Pre-)Release Date](https://img.shields.io/github/release-date-pre/lili-data/metabase-cubejs-driver) | @@ -68,6 +67,7 @@ You install these drivers at your own risk. The plugins will run as part of your | [Dremio](https://github.com/Baoqi/metabase-dremio-driver) | ![GitHub stars](https://img.shields.io/github/stars/Baoqi/metabase-dremio-driver) | ![GitHub (Pre-)Release Date](https://img.shields.io/github/release-date-pre/Baoqi/metabase-dremio-driver) | | [DuckDB](https://github.com//AlexR2D2/metabase_duckdb_driver) | ![GitHub stars](https://img.shields.io/github/stars/AlexR2D2/metabase_duckdb_driver) | ![GitHub (Pre-)Release Date](https://img.shields.io/github/release-date-pre/AlexR2D2/metabase_duckdb_driver) | | [Firebird](https://github.com/evosec/metabase-firebird-driver) | ![GitHub stars](https://img.shields.io/github/stars/evosec/metabase-firebird-driver) | ![GitHub (Pre-)Release Date](https://img.shields.io/github/release-date-pre/evosec/metabase-firebird-driver) | +| [Hydra](https://hydras.io/blog/2022-09-28-metabase-and-hydra) | Hydra connections use the official [Postgres driver](../databases/connections/postgresql.md). | Not applicable. | | [Impala](https://github.com/brenoae/metabase-impala-driver) | ![GitHub stars](https://img.shields.io/github/stars/brenoae/metabase-impala-driver) | ![GitHub (Pre-)Release Date](https://img.shields.io/github/release-date-pre/brenoae/metabase-impala-driver) | | [Materialize](https://github.com/MaterializeInc/metabase-materialize-driver) | ![GitHub stars](https://img.shields.io/github/stars/MaterializeInc/metabase-materialize-driver) | ![GitHub (Pre-)Release Date](https://img.shields.io/github/release-date-pre/MaterializeInc/metabase-materialize-driver) | | [Neo4j](https://github.com/StronkMan/metabase-neo4j-driver) | ![GitHub stars](https://img.shields.io/github/stars/StronkMan/metabase-neo4j-driver) | ![GitHub (Pre-)Release Date](https://img.shields.io/github/release-date-pre/StronkMan/metabase-neo4j-driver) | diff --git a/docs/embedding/full-app-embedding.md b/docs/embedding/full-app-embedding.md index 8dd7c1ef1e78..848096c3799b 100644 --- a/docs/embedding/full-app-embedding.md +++ b/docs/embedding/full-app-embedding.md @@ -8,13 +8,17 @@ redirect_from: {% include plans-blockquote.html feature="Full-app embedding" %} -Metabase offers several [types of embedding](./introduction.md) with different levels of customization and security. +**Full-app embedding** is what you want if you want to offer [multi-tenant, self-service analytics](https://www.metabase.com/learn/customer-facing-analytics/multi-tenant-self-service-analytics). -**Full-app embedding** is the only type of embedding that integrates with your [permissions](../permissions/introduction.md) and [SSO](../people-and-groups/start.md#authentication) to give people the right level of access to [query](https://www.metabase.com/glossary/query_builder) and [drill-down](https://www.metabase.com/learn/questions/drill-through) into your data. +Full-app embedding is the only type of embedding that integrates with your [permissions](../permissions/introduction.md) and [SSO](../people-and-groups/start.md#authentication) to give people the right level of access to [query](https://www.metabase.com/glossary/query_builder) and [drill-down](https://www.metabase.com/learn/questions/drill-through) into your data. -If you only want to set up a fixed number of filters and drill-down views into your data (i.e., prevent people from creating their own [questions](https://www.metabase.com/glossary/question)), you might prefer [Signed embedding](./signed-embedding.md). +## Full-app embedding demo -## Prerequisites +To get a feel for what you can do with full-app embedding, check out our [full-app embedding demo](https://www.metabase.com/embedding-demo). + +To see the query builder in action, click on **Reports** > **+ New** > **Question**. + +## Prerequisites for full-app embedding 1. Make sure you have a [license token](../paid-features/activating-the-enterprise-edition.md) for a [paid plan](https://store.metabase.com/checkout/login-details). 2. Organize people into Metabase [groups](../people-and-groups/start.md). @@ -28,7 +32,7 @@ If you're dealing with a [multi-tenant](https://www.metabase.com/learn/customer- 1. Go to **Settings** > **Admin settings** > **Embedding**. 2. Click **Enable**. 3. Click **Full-app embedding**. -4. Under **Authorized origins**, add the URL of the website or web app where you want to embed Metabase (e.g., `https://*.example.com`). +4. Under **Authorized origins**, add the URL of the website or web app where you want to embed Metabase (such as `https://*.example.com`). ## Setting up embedding on your website @@ -39,10 +43,9 @@ If you're dealing with a [multi-tenant](https://www.metabase.com/learn/customer- - [Add your license token](../configuring-metabase/environment-variables.md#mb_premium_embedding_token). - [Embed Metabase in a different domain](#embedding-metabase-in-a-different-domain). - [Secure your full-app embed](#securing-full-app-embeds). -3. Optional: Enable communication to and from the embedded Metabase using [`postMessage`](https://developer.mozilla.org/en-US/docs/Web/API/Window/postMessage): - - [Fill an entire iframe with an embedded Metabase page](#filling-an-entire-iframe-with-an-embedded-metabase-page). - - [Fit an iframe to a Metabase page with a fixed size](#fitting-an-iframe-to-a-metabase-page-with-a-fixed-size). - - [Pass an embedding URL between Metabase and your app](#passing-an-embedding-url-between-metabase-and-your-app). +3. Optional: Enable communication to and from the embedded Metabase using supported [`postMessage`](https://developer.mozilla.org/en-US/docs/Web/API/Window/postMessage) messages: + - [From Metabase](#supported-postmessage-messages-from-embedded-metabase) + - [To Metabase](#supported-postmessage-messages-to-embedded-metabase) 4. Optional: Set parameters to [show or hide Metabase UI components](#showing-or-hiding-metabase-ui-components). Once you're ready to roll out your full-app embed, make sure that people **allow** browser cookies from Metabase, otherwise they won't be able to log in. @@ -51,17 +54,13 @@ Once you're ready to roll out your full-app embed, make sure that people **allow Go to your Metabase instance and find the page that you want to embed. -For example, to embed your Metabase home page, set the `src` attribute to your [site URL](../configuring-metabase/settings.md#site-url). +For example, to embed your Metabase home page, set the `src` attribute to your [site URL](../configuring-metabase/settings.md#site-url), such as: -``` -http://metabase.yourcompany.com/ -``` +`http://metabase.yourcompany.com/` -To embed a specific Metabase dashboard, use the dashboard's URL: +To embed a specific Metabase dashboard, use the dashboard's URL, such as: -``` -http://metabase.yourcompany.com/dashboard/1 -``` +`http://metabase.yourcompany.com/dashboard/1` ### Pointing an iframe to an authentication endpoint @@ -87,13 +86,11 @@ https://metabase.example.com/auth/sso?jwt=&redirect=%2Fdashboard%2F1%3Ffi ## Embedding Metabase in a different domain -If you want to embed Metabase in another domain (e.g., Metabase is hosted at `metabase.yourcompany.com`, but you want to embed Metabase at `yourcompany.github.io`), set the following [environment variable](../configuring-metabase/environment-variables.md): +If you want to embed Metabase in another domain (say, if Metabase is hosted at `metabase.yourcompany.com`, but you want to embed Metabase at `yourcompany.github.io`), set the following [environment variable](../configuring-metabase/environment-variables.md): -``` -MB_SESSION_COOKIE_SAMESITE=None -``` +`MB_SESSION_COOKIE_SAMESITE=None` -If you set this environment variable to `None`, you must use HTTPS in Metabase to prevent browsers from rejecting the request. For more information, see MDN's documentation on [SameSite cookies](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Set-Cookie/SameSite). +If you set this environment variable to "None", you must use HTTPS in Metabase to prevent browsers from rejecting the request. For more information, see MDN's documentation on [SameSite cookies](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Set-Cookie/SameSite). ## Securing full-app embeds @@ -103,52 +100,44 @@ To limit the amount of time that a person stays logged in, set [`MAX_SESSION_AGE For example, to keep people signed in for 24 hours at most: -``` -MAX_SESSION_AGE=1440 -``` +`MAX_SESSION_AGE=1440` To automatically clear a person's login cookies when they end a browser session: -``` -MB_SESSION_COOKIES=true -``` +`MB_SESSION_COOKIES=true` To manually log someone out of Metabase, load the following URL (for example, in a hidden iframe on the logout page of your application): -``` -https://metabase.yourcompany.com/auth/logout -``` +`https://metabase.yourcompany.com/auth/logout` If you're using [JWT](../people-and-groups/authenticating-with-jwt.md) for SSO, we recommend setting the `exp` (expiration time) property to a short duration (e.g., 1 minute). -## Filling an entire iframe with an embedded Metabase page +## Supported postMessage messages _from_ embedded Metabase -To make an embedded Metabase page fill up the entire iframe (e.g., a question page), use [postMessage](https://developer.mozilla.org/en-US/docs/Web/API/Window/postMessage) to send a "frame" message _from_ Metabase to your app: +To keep up with changes to an embedded Metabase URL (for example, when a filter is applied), set up your app to listen for "location" messages from the embedded Metabase. If you want to use this message for deep-linking, note that "location" mirrors "window.location". ``` -{ “metabase”: { “type”: “frame”, “frame”: { “mode”: “normal” }}} +{ "metabase": { "type": "location", "location": LOCATION_OBJECT_OR_URL }} ``` -## Fitting an iframe to a Metabase page with a fixed size - -To specify the size of an iframe so that it matches an embedded Metabase page (e.g., a dashboard page), use [postMessage](https://developer.mozilla.org/en-US/docs/Web/API/Window/postMessage) to send a "frame" message _from_ Metabase to your app: +To make an embedded Metabase page (like a question) fill up the entire iframe in your app, set up your app to listen for a "frame" message with "normal" mode from Metabase: ``` -{ “metabase”: { “type”: “frame”, “frame”: { “mode”: “fit”, height: HEIGHT_IN_PIXELS }}} +{ "metabase": { "type": "frame", "frame": { "mode": "normal" }}} ``` -## Passing an embedding URL between Metabase and your app - -To make a request for a particular embedding URL (e.g., for deep linking), you can use [postMessage](https://developer.mozilla.org/en-US/docs/Web/API/Window/postMessage) to send a "location" message _from_ your embedded Metabase to your app: +To specify the size of an iframe in your app so that it matches an embedded Metabase page (such as a dashboard), set up your app to listen for a "frame" message with "fit" mode from Metabase: ``` -{ “metabase”: { “type”: “location”, “location”: LOCATION_OBJECT }} +{ "metabase": { "type": "frame", "frame": { "mode": "fit", height: HEIGHT_IN_PIXELS }}} ``` -Or, send a "location" message _to_ your embedded Metabase from your app: +## Supported postMessage messages _to_ embedded Metabase + +To change an embedding URL, send a "location" message from your app to Metabase: ``` -{ “metabase”: { “type”: “location”, “location”: LOCATION_OBJECT_OR_URL }} +{ "metabase": { "type": "location", "location": LOCATION_OBJECT_OR_URL }} ``` ## Showing or hiding Metabase UI components @@ -157,81 +146,65 @@ To change the interface of your full-app embed, you can add parameters to the en For example, you can disable Metabase's [top nav bar](#top_nav) and [side nav menu](#side_nav) like this: -``` -your_embedding_url?top_nav=false&side_nav=false -``` +`your_embedding_url?top_nav=false&side_nav=false` ![Top nav and side nav disabled](./images/no-top-no-side.png) -### `top_nav` +### top_nav Hidden by default. To show the top navigation bar: -``` -top_nav=true -``` +`top_nav=true` ![Top nav bar](./images/top-nav.png) -### `search` +### search Hidden by default. To show the search box in the top nav: -``` -top_nav=true&search=true -``` +`top_nav=true&search=true` -### `new_button` +### new_button Hidden by default. To show the **+ New** button used to create queries or dashboards: -``` -top_nav=true&new_button=true -``` +`top_nav=true&new_button=true` -### `side_nav` +### side_nav The navigation sidebar is shown on `/collection` and home page routes, and hidden everywhere else by default. To allow people to minimize the sidebar: -``` -top_nav=true&side_nav=true -``` +`top_nav=true&side_nav=true` ![Side nav](./images/side-nav.png) -### `header` +### header Visible by default on question and dashboard pages. To hide a question or dashboard's title, [additional info](#additional_info), and [action buttons](#action_buttons): -``` -header=false -``` +`header=false` -### `additional_info` +### additional_info Visible by default on question and dashboard pages, when the [header](#header) is enabled. -To hide the gray text “Edited X days ago by FirstName LastName”, as well as the breadcrumbs with collection, database, and table names: +To hide the gray text "Edited X days ago by FirstName LastName", as well as the breadcrumbs with collection, database, and table names: -``` -header=false&additional_info=false -``` +`header=false&additional_info=false` ![Additional info](./images/additional-info.png) -### `action_buttons` +### action_buttons Visible by default on question pages when the [header](#header) is enabled. -To hide the action buttons such as **Save**, **Summarize**, **Filter**, or the query builder icon: +To hide the action buttons such as **Filter**, **Summarize**, the query builder button, and so on: -``` -header=false&action_buttons=false -``` +`header=false&action_buttons=false` ![Action buttons](./images/action-buttons.png) diff --git a/docs/embedding/introduction.md b/docs/embedding/introduction.md index 42f54e24cccc..6413e5dd9fb8 100644 --- a/docs/embedding/introduction.md +++ b/docs/embedding/introduction.md @@ -8,28 +8,42 @@ redirect_from: You can embed Metabase tables, charts, and dashboards—even Metabase's query builder—in your website or application. -[Signed embedding](./signed-embedding.md) (also known as standalone embedding) and [full-app embedding](./full-app-embedding.md) are _secure_ ways to share your data with specific groups of people outside of your organization. +## Different ways to embed -If you'd like to share your data with the good people of the internet, you can create a [public link](../questions/sharing/public-links.md) and embed that directly on your website. +There are three ways to embed Metabase in your app: -## How embedding works +- [Full-app embedding](#full-app-embedding) +- [Signed embedding](#signed-embedding) +- [Public links and embeds](#public-links-and-embeds) -You'll need to put an iframe on your website to act as a window to your Metabase app. Different configurations of that embedded iframe will let you: +## Full-app embedding -- [set up public access](../questions/sharing/public-links.md) to charts and dashboards, -- [require sign-in](./signed-embedding.md) to view personalized versions of those charts and dashboards, or -- [integrate with SSO and data permissions](./full-app-embedding.md) to enable self-service access to the underlying data. +Full-app embedding is the only kind of embedding that [integrates with SSO and data permissions](./full-app-embedding.md) to enable true self-service access to the underlying data. + +**When to use full-app embedding**: when you want to [offer multi-tenant, self-service analytics](https://www.metabase.com/blog/why-full-app-embedding). With full-app embedding, people can create their own questions, dashboards, models, and more, all in their own data sandbox. + +## Signed embedding + +Also known as standalone embedding, signed embedding is a secure way to embed charts and dashboards. + +**When to use signed embedding**: you don’t want to give people ad hoc query access to their data for whatever reason, or you want to present data that applies to all of your tenants at once. For example, say you want to showcase some benchmarking stats: if you just want to make those stats available exclusively to your customers, you could use a signed embed. + +## Public links and embeds + +If you'd like to share your data with the good people of the internet, you can create a [public link](../questions/sharing/public-links.md) or embed a question or dashboard directly in your website. + +**When to use public links and embeds**: public links and embeds are good for one-off charts and dashboards. Use them when you just need to show someone a chart or dashboard without giving people access to your Metabase. And you don't care who sees the data; you want to make those stats available to everyone. ## Comparison of embedding types -| | [Public](../questions/sharing/public-links.md) | [Signed](./signed-embedding.md) | [Full-app](./full-app-embedding.md) | -| -------------------------------------------------------------------------------------------------------- | ---------------------------------------------- | ------------------------------- | ----------------------------------- | -| Display charts and dashboards | ✅ | ✅ | ✅ | -| Display interactive [filter widgets](https://www.metabase.com/glossary/filter_widget) | ✅ | ✅ | ✅ | -| Restrict data with [locked filters](./signed-embedding-parameters.md#restricting-data-in-a-signed-embed) | ❌ | ✅ | ❌ | -| Restrict data with [sandboxes](../permissions/data-sandboxes.md) | ❌ | ❌ | ✅ | -| Drill-down using the [action menu](https://www.metabase.com/learn/questions/drill-through) | ❌ | ❌ | ✅ | -| Self-serve via [query builder](https://www.metabase.com/glossary/query_builder) | ❌ | ❌ | ✅ | +| Action | [Full-app](./full-app-embedding.md) | [Signed](./signed-embedding.md) | [Public](../questions/sharing/public-links.md) | +|----------------------------------------------------------------------------------------------------------|-------------------------------------|---------------------------------|------------------------------------------------| +| Display charts and dashboards | ✅ | ✅ | ✅ | +| Display interactive [filter widgets](https://www.metabase.com/glossary/filter_widget) | ✅ | ✅ | ✅ | +| Restrict data with [locked filters](./signed-embedding-parameters.md#restricting-data-in-a-signed-embed) | ❌ | ✅ | ❌ | +| Restrict data with [sandboxes](../permissions/data-sandboxes.md) | ✅ | ❌ | ❌ | +| Drill-down using the [action menu](https://www.metabase.com/learn/questions/drill-through) | ✅ | ❌ | ❌ | +| Self-serve via [query builder](https://www.metabase.com/glossary/query_builder) | ✅ | ❌ | ❌ | ## Further reading diff --git a/docs/embedding/signed-embedding-parameters.md b/docs/embedding/signed-embedding-parameters.md index f20a17bd3f80..2b9f81e255c5 100644 --- a/docs/embedding/signed-embedding-parameters.md +++ b/docs/embedding/signed-embedding-parameters.md @@ -43,13 +43,13 @@ For example, if your embedded dashboard has a filter called "Breakfast", and you your_embedding_url?breakfast=Scrambled_eggs ``` -To specify default values for more than one filter, separate them with ampersands (&): +To specify default values for more than one filter, separate the filters with ampersands (&): ``` your_embedding_url?breakfast=Scrambled_eggs&lunch=Grilled_cheese ``` -If the original dashboard's filter widget accepts multiple values (i.e., it's a [dropdown filter](../dashboards/filters.md#choosing-between-a-dropdown-or-autocomplete-for-your-filter), not a text box filter), you can set multiple default values: +You can set multiple default values for a filter by separating the `key=value` pairs with ampersands (&): ``` your_embedding_url?breakfast=Scrambled_eggs&breakfast=Bacon @@ -71,14 +71,11 @@ You can use locked parameters to display filtered data based on attributes captu Locked parameters will apply the selected filter values to your original dashboard or SQL question, but they won't be displayed as filter widgets on your embed. Locked parameters may also limit the values that are shown in your [embedded filter widgets](#adding-a-filter-widget-to-a-signed-embed). -## Adding multiple values to a locked parameter +## Locked parameters on dashboards with SQL questions -If your [locked parameter](#restricting-data-in-a-signed-embed) is linked to a: +If your [locked parameter](#restricting-data-in-a-signed-embed) is linked to a dashboard filter that's in turn linked to a SQL question, you'll only be able to choose a _single_ value for your locked parameter. -- dashboard filter, change the filter type to a [dropdown filter](../dashboards/filters.md#choosing-between-a-dropdown-or-autocomplete-for-your-filter). -- SQL question, [change the SQL variable type](../questions/native-editor/sql-parameters.md#filter-widget-with-dropdown-menu-and-search). - -For example, let's say you have a text box filter on your original dashboard called "Breakfast" with the values "Scrambled eggs", "Bacon", and "Waffles". With this setup, you'll be able to choose _one_ of "Scrambled eggs", "Bacon", or "Waffles" for a locked parameter linked to the "Breakfast" filter. If you want to provide the values "Scrambled eggs" _and_ "Bacon to the locked parameter, change the original "Breakfast" filter on your dashboard to a dropdown filter. +For example, let's say you have a dashboard filter called "Breakfast" with the values "Scrambled eggs", "Bacon", and "Waffles". If the "Breakfast" filter is linked to _any_ SQL questions on the dashboard, you'll only be able to choose _one_ of "Scrambled eggs", "Bacon", or "Waffles" for a locked parameter linked to the "Breakfast" filter. ## Hiding filter widgets from a signed embed @@ -104,25 +101,27 @@ your_embedding_url?breakfast=Scrambled_eggs#hide_parameters=breakfast ## Customizing the appearance of a signed embed -You can change the appearance of an embedded item by adding parameters with the following values: +You can change the appearance of an embedded item by adding hash parameters to the end of the URL in your iframe's `src` attribute. + +For example, the following embedding URL will display an embedded item in dark mode, without a border, and with its original title: + +``` +your_embedding_url#theme=night&bordered=false&titled=true +``` + +You can preview appearance settings from your question or dashboard's [embedded appearance settings](./signed-embedding.md#customizing-the-appearance-of-signed-embeds). | Parameter name | Possible values | | ---------------------- | --------------------------------------------- | | bordered | true, false | | titled | true, false | | theme | null, transparent, night | -| font\* | [font name](../configuring-metabase/fonts.md) | -| hide_download_button\* | true, false | - -\* Available on paid plans. - -You can preview the changes from your question or dashboard's [embedded appearance settings](./signed-embedding.md#customizing-the-appearance-of-signed-embeds). +| font¹ | [font name](../configuring-metabase/fonts.md) | +| hide_download_button² | true, false | -For example, the following embedding URL will display an embedded item in dark mode, with its original title, and without a border: +¹ Available on [paid plans](https://www.metabase.com/pricing). -``` -your_embedding_url#theme=night&titled=true&bordered=false -``` +² Available on [paid plans](https://www.metabase.com/pricing) and works on questions only (not dashboards). ## Further reading diff --git a/docs/embedding/signed-embedding.md b/docs/embedding/signed-embedding.md index b8e14a29a93d..6d3621dd8097 100644 --- a/docs/embedding/signed-embedding.md +++ b/docs/embedding/signed-embedding.md @@ -108,15 +108,17 @@ You can change the way an embedded question or dashboard looks in an iframe (whi - Border - Title - Theme (light, dark, transparent) -- Font\* -- Download data\* (Note: This setting only hides the download button. To disable the feature entirely, use the [Download results permission](../permissions/data.md#download-results).) +- Font¹ +- Download data² -\* Available on paid plans. +¹ Available on [paid plans](https://www.metabase.com/pricing). + +² Available on [paid plans](https://www.metabase.com/pricing) and hides the download button on questions only (not dashboards). To update the appearance of a signed embed: 1. Optional: Preview the appearance changes from your question or dashboard's embedding settings (**sharing icon** > **Embed this item in an application**). -2. Optional: Click **Code** to find the updated server code snippet in the top block. +2. Optional: Click **Code** to find the updated server code snippet in the top code block. 3. Change the [parameters](./signed-embedding-parameters.md#customizing-the-appearance-of-a-signed-embed) in your actual server code. For global appearance settings, such as the colors and fonts used across your entire Metabase instance, see [Customizing Metabase's appearance](../configuring-metabase/appearance.md). diff --git a/docs/embedding/start.md b/docs/embedding/start.md index a290c2205b49..6dd3faebbc80 100644 --- a/docs/embedding/start.md +++ b/docs/embedding/start.md @@ -8,10 +8,15 @@ title: Embedding overview What is embedding, and how does it work? +## [Full-app embedding](./full-app-embedding.md) + +The solution to self-service customer analytics: embed the full Metabase app in your app. Full-app embedding integrates with your data permissions to let people slice and dice data on their own using Metabase's query builder. + ## [Signed embedding](./signed-embedding.md) Also known as Standalone Embedding, Signed Embedding offers drill-through with custom destinations, so you can define what happens when people click on a chart, like sending people to another chart or URL–all while securing the underlying data. -## [Full-app embedding](./full-app-embedding.md) +## [Parameters for signed embeds](./signed-embedding-parameters.md) + +You can pass parameters between Metabase and your website via the embedding URL to specify how Metabase items should look and behave inside the iframe on your website. -Put all of Metabase in your app to give people secure self-serve access to data. You can still create custom destinations if you want, but Full-app Embedding integrates with your data permissions to let people slice and dice data on their own using Metabase's query builder. diff --git a/docs/exploration-and-organization/collections.md b/docs/exploration-and-organization/collections.md index b637dbed1603..4982fd09d2b3 100644 --- a/docs/exploration-and-organization/collections.md +++ b/docs/exploration-and-organization/collections.md @@ -58,14 +58,6 @@ To move a question, dashboard, or pulse into a collection, or from one collectio Note that you have to have Curate permission for the collection that you're moving a question into _and_ the collection you're moving the question out of. -## Archiving items - -Sometimes questions outlive their usefulness and need to be sent to Question Heaven. To archive a question or dashboard, just click on the `…` menu that appears on the far right when you hover over a question and pick the Archive action. You'll only see that option if you have "curate" permission for the current collection. You can also archive multiple items at once, the same way as you move multiple items. Note that archiving a question removes it from all dashboards or Pulses where it appears, so be careful! - -You can also archive _collections_ if you have curate permissions for the collection you're trying to archive, the collection _it's_ inside of, as well as any and all collections inside of _it_. Archiving a collection archives all of its contents as well. - -If you have second thoughts and want to bring an archived item back, you can see all your archived questions from the archive; click the menu icon in the top-right of any collection page to get to the archive. To unarchive a question, hover over it and click the unarchive icon that appears on the far right. - ## Events and timelines You can add events to collections, and organize those events into timelines. See [Events and timelines](events-and-timelines.md). diff --git a/docs/exploration-and-organization/data-model-reference.md b/docs/exploration-and-organization/data-model-reference.md new file mode 100644 index 000000000000..8b7b6ef6703f --- /dev/null +++ b/docs/exploration-and-organization/data-model-reference.md @@ -0,0 +1,39 @@ +--- +title: Data reference +redirect_from: + - /docs/latest/users-guide/12-data-model-reference + - /docs/latest/questions/native-editor/data-model-reference +--- + +# Data reference + +Sometimes when you're composing a question, you might forget the exact names of different tables or columns, or orwhiceh table contains what. That’s where the **Data reference** comes in handy. + +You can open the data reference from: + +- The Metabase sidebar > **Browse data** > **Learn about our data**. +- The SQL editor > **book** icon. + +## Browse data reference + +The data reference is a collection of pages organized by database, then tables in that database (the same structure you use to [pick data](../questions/query-builder/introduction.md#picking-data)). The data reference section is the best place to find information about the [data types](https://www.metabase.com/learn/databases/data-types-overview) of columns in your data. + +![Data reference page](./images/data-reference-page.png) + +## SQL editor data reference + +This panel lists all the databases you have access to, and the [models](../data-modeling/models.md) and tables, and the fields inside of them. + +![Data reference sidebar](./images/DataReference.png) + +Click on the model or table you would like to explore further to see a description of it and a list of all the columns it has. Each table or column will only contain a description if your admin wrote something in the Data Model section of the Admin Panel. + +If you click on a column you’re interested in, you’ll see a description of its contents, as well as a list of sample values for columns that don't have a huge number of distinct values. + +### Connections + +![Connections](./images/connections.png) + +In the data reference sidebar, under **Connections**, you can find a list of table relationships (defined by [foreign keys](https://www.metabase.com/glossary/foreign_key)). + +If a foreign key relationship exists in your database, but doesn't show up in your Metabase, your Metabase admin will need to update your [Metabase data model](../data-modeling/metadata-editing.md). diff --git a/docs/exploration-and-organization/exploration.md b/docs/exploration-and-organization/exploration.md index 79d5130a36a2..281b36ce0398 100644 --- a/docs/exploration-and-organization/exploration.md +++ b/docs/exploration-and-organization/exploration.md @@ -96,6 +96,18 @@ Some things to remember with bookmarks: - Items that you bookmark will get a boost in your search results (but not the search results of other people). - To reorder bookmarks, simply drag and drop them in the sidebar. +## Verified items + +{% include plans-blockquote.html feature="Verification" %} + +Verified questions and models are marked with a blue checkmark icon: + +![Verified icon](./images/verified-icon.png) + +Administrators can **Verify** a question or model from the three dot menu (`...`) to signal that they've reviewed the item and deemed it to be trustworthy. That is: the question or model is filtering the right columns, summarizing the right metrics, and querying records from the right tables. Verified items are more likely to show up higher in search suggestions and search results. + +If someone modifies a verified question, the question will lose its verified status, and an administrator will need to review and verify the question again to restore its verified status. + [collections]: ./collections.md [dashboards]: ../dashboards/start.md [models]: ../data-modeling/models.md diff --git a/docs/exploration-and-organization/history.md b/docs/exploration-and-organization/history.md new file mode 100644 index 000000000000..faa76bb09d1d --- /dev/null +++ b/docs/exploration-and-organization/history.md @@ -0,0 +1,94 @@ +--- +title: History +--- + +# History + +For questions, dashboards, and models, Metabase keeps a version history for the previous fifteen versions of that item. You can view changes, and revert to previous versions. + +You can also archive, unarchive, and permanently delete outdated items. + + +## Viewing tracked changes + +1. Go to your question, dashboard, or model. +2. Click the info icon. +3. A sidebar will pop up with a history of up to 15 versions. + +Metabase will keep track of a version each time you [save](../questions/sharing/answers.md#how-to-save-a-question), [move](../questions/sharing/answers.md#editing-your-question), [revert](#reverting-to-previous-versions), [archive](#archiving-items), or [verify](./exploration.md#verified-items) an item. + +## Reverting to previous versions + +1. Go to your question, dashboard, or model. +2. Click the info icon (an **i** in a circle). +3. A sidebar will appear with up to fifteen previous versions. +4. Click on the **back arrow** beside a version to revert your item to that point in time. + +## Archiving items + +Sometimes your questions, dashboards, or models outlive their usefulness. You can send outdated items to the **Archive** from the three dot menu `...` of your questions, dashboards, or models. + +Note that archiving an item will affect any [dashboards](../dashboards/introduction.md), [subscriptions](../dashboards/subscriptions.md), or [SQL questions](../questions/native-editor/referencing-saved-questions-in-queries.md) that depend on that item, so be careful! + +### Archiving multiple items + +You can batch archive multiple items from the same collection: + +1. Go to the collection. +2. Hover over the icon beside the name of the item. +3. Click the checkbox that appears. +4. When you're done selecting your items, click **Archive** at the bottom of the page. + +### Archiving a collection + +1. Go to the collection. +2. Click `...` > **Archive**. + +Archiving a collection archives all of the collection's contents as well. You can only archive a collection if you have permission to **Curate** a collection (as well as all of the collections inside that collection). If you don't see the **Archive** option, see your Metabase admin about your [collection permissions](../permissions/collections.md). + +### Viewing the archive + +1. Open the main Metabase sidebar. +2. Click the `...` beside the "Collections" header in the sidebar. +3. Click **View archive**. + +## Unarchiving items + +1. Open the main Metabase sidebar. +2. Click the `...` beside the "Collections" header in the sidebar. +3. Click **View archive**. +4. Hover over the item and click the **unarchive** icon (rectangle with a `^` symbol). + +The unarchived item should get restored to the parent collection that it was most recently saved in. If you unarchive an item, but you don't know where it's reappeared: + +- search for the item directly, or +- check if the item's parent collection is also in the archive. + +### Unarchiving multiple items + +You can unarchive multiple items at once from the same collection: + +1. Go to the collection. +2. Hover over the icon beside the name of the item and click the checkbox that appears. +3. When you're done selecting your items, click **Unarchive** at the bottom of the page. + +## Deleting items permanently + +1. Open the main Metabase sidebar. +2. Click the `...` beside the "Collections" header in the sidebar. +3. Click **View archive**. +4. Hover over the item and click the **trash bin** icon. + +The item will get permanently deleted from your application database. + +Remember that [archiving](#archiving-items) and deleting items can have unanticipated ripple effects on related [dashboards](../dashboards/introduction.md), [subscriptions](../dashboards/subscriptions.md), and [SQL questions](../questions/native-editor/referencing-saved-questions-in-queries.md). + +We recommend archiving because you can always unarchive if something breaks. If you delete an item and accidentally break something, you might have to recreate all of that work from scratch (unless you're prepared to revert to a backup of your application database). + +### Deleting multiple items permanently + +You can delete multiple items at once from the same collection: + +1. Go to the collection. +2. Hover over the icon beside the name of the item and click the checkbox that appears. +3. When you're done selecting your items, click **Delete** at the bottom of the page. diff --git a/docs/exploration-and-organization/images/DataReference.png b/docs/exploration-and-organization/images/DataReference.png new file mode 100644 index 000000000000..494ce17791f2 Binary files /dev/null and b/docs/exploration-and-organization/images/DataReference.png differ diff --git a/docs/exploration-and-organization/images/connections.png b/docs/exploration-and-organization/images/connections.png new file mode 100644 index 000000000000..99dace9a040d Binary files /dev/null and b/docs/exploration-and-organization/images/connections.png differ diff --git a/docs/questions/images/data-reference-column-detail.png b/docs/exploration-and-organization/images/data-reference-column-detail.png similarity index 100% rename from docs/questions/images/data-reference-column-detail.png rename to docs/exploration-and-organization/images/data-reference-column-detail.png diff --git a/docs/exploration-and-organization/images/data-reference-page.png b/docs/exploration-and-organization/images/data-reference-page.png new file mode 100644 index 000000000000..f03f2c025c7c Binary files /dev/null and b/docs/exploration-and-organization/images/data-reference-page.png differ diff --git a/docs/questions/images/verified-icon.png b/docs/exploration-and-organization/images/verified-icon.png similarity index 100% rename from docs/questions/images/verified-icon.png rename to docs/exploration-and-organization/images/verified-icon.png diff --git a/docs/exploration-and-organization/start.md b/docs/exploration-and-organization/start.md index 7c9d14f28d58..2692662d0741 100644 --- a/docs/exploration-and-organization/start.md +++ b/docs/exploration-and-organization/start.md @@ -14,6 +14,14 @@ Find data, explore questions and dashboards, and bookmark your favorites. Organize questions, dashboards, and models with collections. +## [History](./history.md) + +View changes to a question, dashboard or model, revert to previous versions, and archive outdated items. + +## [Data reference](./data-model-reference.md) + +Documentation that helps you understand your data sources: tables, columns, metrics, and more. + ## [Events and timelines](./events-and-timelines.md) Add events to timelines to annotate charts. diff --git a/docs/installation-and-operation/images/prometheus.png b/docs/installation-and-operation/images/prometheus.png new file mode 100644 index 000000000000..6c511afd2591 Binary files /dev/null and b/docs/installation-and-operation/images/prometheus.png differ diff --git a/docs/installation-and-operation/monitoring-metabase.md b/docs/installation-and-operation/monitoring-metabase.md index 7c6dc667cd26..1eb650fd76b6 100644 --- a/docs/installation-and-operation/monitoring-metabase.md +++ b/docs/installation-and-operation/monitoring-metabase.md @@ -7,6 +7,7 @@ redirect_from: # Monitoring Your Metabase + Diagnosing performance related issues can be a challenge. Luckily the JVM ships with tools that can help diagnose many common issues. Enabling JMX and using a tool like VisualVM can help diagnose issues related to running out of memory, a stalled Metabase instance, and slow response times. This guide assumes that you have the VisualVM tool installed @@ -120,3 +121,8 @@ that specific point in time. Collect a thread dump via the "Threads" tab: ![threaddump](images/ThreadDump.png) + + +## Further reading + +- [Observability with Prometheus](./observability-with-prometheus.md) diff --git a/docs/installation-and-operation/observability-with-prometheus.md b/docs/installation-and-operation/observability-with-prometheus.md new file mode 100644 index 000000000000..5d76ad78251a --- /dev/null +++ b/docs/installation-and-operation/observability-with-prometheus.md @@ -0,0 +1,161 @@ +--- +title: observability-with-prometheus +--- + +# Observability with Prometheus + +You can export metrics in [Prometheus](https://prometheus.io/) format from your Metabase. + +## Running Metabase and Prometheus locally + +To give you an idea of how Metabase and Prometheus would work in your production environment, we'll walk through how to set up Metabase and Prometheus locally. + +### Start up Metabase with `MB_PROMETHEUS_SERVER_PORT` + +Download the latest [Metabase JAR](https://www.metabase.com/start/oss/), and run Metabase using an environment variable to specify the Prometheus server port: + +``` +MB_PROMETHEUS_SERVER_PORT=9191 java -jar metabase.jar +``` + +The `MB_PROMETHEUS_SERVER_PORT=9191` specifies which port (`9191`) Metabase will use to send data to Prometheus. To clarify the ports that will be involved here: + +- Port `3000` is the port Metabase uses to serve the Metabase app. You can set another port with `MB_JETTY_PORT` (e.g., `MB_JETTY_PORT=3001`). + - Port `9191` (or whichever port you specified with the `MB_PROMETHEUS_SERVER_PORT` environment variable) is the port Prometheus uses to scrape metrics from Metabase. +- Port `9090` is the port Prometheus uses to serve the Prometheus application. + +When you start Metabase, the Metabase logs will tell you that Metabase is starting the `prometheus metrics collector` and `prometheus metrics web-server`. + +``` +(truncated logs) +2022-09-01 17:47:38,808 INFO metabase.util :: Database setup took 3.4 s +2022-09-01 17:47:38,826 INFO metabase.core :: Setting up prometheus metrics +2022-09-01 17:47:38,827 INFO metabase.prometheus :: Starting prometheus metrics collector +2022-09-01 17:47:38,839 INFO metabase.prometheus :: Starting prometheus metrics web-server on port 9,191 +(truncated logs) +``` + +You can view your locally running Metabase at `http://localhost:3000`. + +### Download and configure Prometheus + +[Download Prometheus](https://prometheus.io/download), and extract the files. + +Change into the Prometheus directory, add the following YAML file to configure your Prometheus: + +#### Prometheus configuration file example + +``` +global: + scrape_interval: 15s # By default, scrape targets every 15 seconds. + + # Attach these labels to any time series or alerts when communicating with + # external systems (federation, remote storage, Alertmanager). + external_labels: + monitor: 'codelab-monitor' + +# A scrape configuration containing exactly one endpoint to scrape: +# Here it's Prometheus itself. +scrape_configs: + # The job name is added as a label `job=` to any timeseries scraped from this config. + - job_name: 'prometheus' + + # Override the global default and scrape targets from this job every 5 seconds. + scrape_interval: 5s + # use whatever port here that you set for MB_PROMETHEUS_SERVER_PORT + static_configs: + - targets: ['localhost:9191'] +``` + +### Running Prometheus Locally + +In a new terminal process in the Prometheus directory, run: + +``` +./prometheus --config.file=prometheus.yml +``` + +Then check `http://localhost:9090`. You should see the Prometheus app, and be able to search for various metrics emitted by Metabase. + +![Prometheus page showing `jvm_thread_state` graph](./images/prometheus.png) + +## Sample metrics output + +Here is some sample output from Metabase: + +``` +'# HELP jvm_threads_current Current thread count of a JVM +'# TYPE jvm_threads_current gauge +jvm_threads_current 81.0 +'# HELP jvm_threads_daemon Daemon thread count of a JVM +'# TYPE jvm_threads_daemon gauge +jvm_threads_daemon 36.0 +'# HELP jvm_threads_peak Peak thread count of a JVM +'# TYPE jvm_threads_peak gauge +jvm_threads_peak 81.0 +'# HELP jvm_threads_started_total Started thread count of a JVM +'# TYPE jvm_threads_started_total counter +jvm_threads_started_total 104.0 +'# HELP jvm_threads_deadlocked Cycles of JVM-threads that are in deadlock waiting to acquire object monitors or ownable synchronizers +'# TYPE jvm_threads_deadlocked gauge +jvm_threads_deadlocked 0.0 +``` + +## Exported metrics + +Metrics exported by Metabase include: + +- `c3p0_max_pool_size` +- `c3p0_min_pool_size` +- `c3p0_num_busy_connections` +- `c3p0_num_connections` +- `c3p0_num_idle_connections` +- `jetty_async_dispatches_total` +- `jetty_async_requests_total` +- `jetty_async_requests_waiting` +- `jetty_async_requests_waiting_max` +- `jetty_dispatched_active` +- `jetty_dispatched_active_max` +- `jetty_dispatched_time_max` +- `jetty_dispatched_time_seconds_total` +- `jetty_dispatched_total` +- `jetty_expires_total` +- `jetty_request_time_max_seconds` +- `jetty_request_time_seconds_total` +- `jetty_requests_active` +- `jetty_requests_active_max` +- `jetty_requests_total` +- `jetty_responses_bytes_total` +- `jetty_responses_total` +- `jetty_stats_seconds` +- `jvm_gc_collection_seconds_count` +- `jvm_gc_collection_seconds_sum` +- `jvm_memory_bytes_committed` +- `jvm_memory_bytes_init` +- `jvm_memory_bytes_max` +- `jvm_memory_bytes_used` +- `jvm_memory_objects_pending_finalization` +- `jvm_memory_pool_bytes_committed` +- `jvm_memory_pool_bytes_init` +- `jvm_memory_pool_bytes_max` +- `jvm_memory_pool_bytes_used` +- `jvm_memory_pool_collection_committed_bytes` +- `jvm_memory_pool_collection_init_bytes` +- `jvm_memory_pool_collection_max_bytes` +- `jvm_memory_pool_collection_used_bytes` +- `jvm_threads_current` +- `jvm_threads_daemon` +- `jvm_threads_deadlocked` +- `jvm_threads_deadlocked_monitor` +- `jvm_threads_peak` +- `jvm_threads_started_total` +- `jvm_threads_state` +- `process_cpu_seconds_total` +- `process_max_fds` +- `process_open_fds` +- `process_start_time_seconds` + + +## Further reading + +- [Monitoring Metabase](./monitoring-metabase.md) diff --git a/docs/installation-and-operation/running-metabase-on-heroku.md b/docs/installation-and-operation/running-metabase-on-heroku.md index 0c01dd658e33..3c7e567be1bb 100644 --- a/docs/installation-and-operation/running-metabase-on-heroku.md +++ b/docs/installation-and-operation/running-metabase-on-heroku.md @@ -6,7 +6,10 @@ redirect_from: # Running Metabase on Heroku -Currently in beta. We've run Metabase on Heroku and it works just fine, but it's not hardened for production use just yet. If you're up for it then give it a shot and let us know how we can make it better! + +> Note: Metabase will be deprecating Heroku in an upcoming release. To migrate off Heroku, see our guide on [Migrating from Heroku to Metabase Cloud](https://www.metabase.com/cloud/docs/migrate/heroku). + +We've run Metabase on Heroku and it works just fine, but it's not hardened for production. ### Launching Metabase diff --git a/docs/installation-and-operation/start.md b/docs/installation-and-operation/start.md index dea14a1cce0b..9a1adf3ba5a3 100644 --- a/docs/installation-and-operation/start.md +++ b/docs/installation-and-operation/start.md @@ -54,6 +54,10 @@ How to use an RDS instance as your application database. Monitor your Metabase with JMX. +## [Observability with Prometheus](./observability-with-prometheus.md) + +Export Metabase metrics for viewing with Prometheus. + ## [Java versions](./java-versions.md) Some notes on Java versions. diff --git a/docs/paid-features/activating-the-enterprise-edition.md b/docs/paid-features/activating-the-enterprise-edition.md index bae72294a5da..f13811f908ed 100644 --- a/docs/paid-features/activating-the-enterprise-edition.md +++ b/docs/paid-features/activating-the-enterprise-edition.md @@ -6,14 +6,18 @@ redirect_from: # Activating your Metabase commercial license -The [paid Pro and Enterprise editions](https://www.metabase.com/pricing) of Metabase are distinct from the free [Open Source edition](../installation-and-operation/running-the-metabase-jar-file.md), so to use your paid features you’ll need to first get a license. And if you want to self-host, you'll need a different JAR or Docker image that you can use to activate the advanced features with your license token. +The [paid Pro and Enterprise editions](https://www.metabase.com/pricing) of Metabase are distinct from the free [Open Source edition](../installation-and-operation/running-the-metabase-jar-file.md). + +To use your paid features you’ll need to first get a license. And if you want to self-host, you'll need a different JAR or Docker image that you can use to activate the advanced features with your license token. + +## If you're running on Metabase Cloud + +If you've signed up for or upgraded to a premium plan on Metabase Cloud, all of this will be taken care of for you. ## Where to get a license You can get a license by signing up for a free trial of the [Pro or Enterprise edition plans](https://www.metabase.com/pricing), both of which can be self-hosted or hosted on Metabase Cloud. -If you sign up for a Metabase Cloud option, you're already good to go. - ## How to activate your token when self-hosting If you chose to host Metabase yourself, you'll get an email containing a unique license token. But to use it, you'll need to install the right JAR file. @@ -23,13 +27,21 @@ You can either: - [Download the latest metabase-enterprise JAR](https://downloads.metabase.com/enterprise/latest/metabase.jar) (the filename is the same, irrespective of your plan), or - [Get the latest Docker image](https://hub.docker.com/r/metabase/metabase-enterprise/) at `metabase/metabase-enterprise:latest`. -Run Metabase as you would normally, then go to __Settings__ > __Admin settings__, and click __License and Billing__ in the lefthand sidebar. Paste in your license token under __License__ and click __Activate__. +Run Metabase as you would normally, then go to **Settings** > **Admin settings**, and click **License and Billing** in the lefthand sidebar. Paste in your license token under **License** and click **Activate**. ## **Validating your token** -Your Metabase needs to be able to access the internet (specifically `https://store.metabase.com/api/[token-id]/v2/status`) in order to validate the token and maintain access to the advanced features. +To validate your token and maintain access to paid features, your Metabase needs to be able to access the Internet, specifically: + +``` +https://token-check.metabase.com/api/[token-id]/v2/status +``` + +(substituting `[token-id]` with your token ID). + +If your Metabase can't validate the token, it'll disable the paid features, but will continue to work normally as if you were running the Open Source edition. -If your Metabase can't validate the token, it'll disable the advanced features, but will continue to work normally otherwise, as if it were the Open Source edition. +## Routing outbound Metabase traffic through a proxy In case you need to route outbound Metabase traffic through a proxy on your network, use the following command when starting Metabase: @@ -37,4 +49,14 @@ In case you need to route outbound Metabase traffic through a proxy on your netw java -Dhttps.proxyHost=[your proxy's hostname] -Dhttps.proxyPort=[your proxy's port] -jar enterprise_metabase.jar ``` -Depending on your organization’s setup, you may need to take additional configuration steps. If the command above doesn't work for you, we recommend reaching out to your internal infrastructure or dev ops teams for assistance. \ No newline at end of file +Depending on your organization’s setup, you may need to take additional configuration steps. If the command above doesn't work for you, we recommend reaching out to your internal infrastructure or dev ops teams for assistance. + +## IP addresses to whitelist + +If you're hosting Metabase behind a firewall that blocks outgoing connections, you'll need to allow these IP addresses to ensure access to `token-check.metabase.com` to verify your license. + +``` +23.23.111.13 +44.199.18.109 +44.212.138.188 +``` diff --git a/docs/paid-features/overview.md b/docs/paid-features/overview.md index 05371cbf0599..7f79f2f7fab7 100644 --- a/docs/paid-features/overview.md +++ b/docs/paid-features/overview.md @@ -6,7 +6,7 @@ redirect_from: # Overview of premium features -Metabase's [Enterprise and Pro][pricing] plans provide additional features that help organizations scale Metabase and deliver self-service, embedded analytics. +Metabase's [Enterprise and Pro](https://www.metabase.com/pricing) plans provide additional features that help organizations scale Metabase and deliver self-service, embedded analytics. ## Setting up @@ -54,9 +54,9 @@ You can mark certain collections as [official](../exploration-and-organization/c ## Question moderation -People can ask administrators to verify their questions. +People can ask administrators to verify their questions and models. -- [Question moderation](../questions/sharing/answers.md#question-moderation) +- [Verified items](../exploration-and-organization/exploration.md#verified-items) ## Advanced caching controls @@ -82,4 +82,6 @@ You can export Metabase application data and use that to spin up new instances p - [Serialization](../installation-and-operation/serialization.md) -[pricing]: https://www.metabase.com/pricing +## Configuration file + +For self-hosted installations, you can load Metabase from a [configuration file](../configuring-metabase/config-file.md). diff --git a/docs/people-and-groups/managing.md b/docs/people-and-groups/managing.md index d45cf2310071..cc1178dd6a85 100644 --- a/docs/people-and-groups/managing.md +++ b/docs/people-and-groups/managing.md @@ -10,25 +10,33 @@ To start managing people, click on the **gear** icon > **Admin settings** > **Pe ![Admin menu](images/AdminBar.png) -## Managing people +## Creating an account -### Creating accounts for your team +To add a new person, click **Invite someone** in the upper right corner. You’ll be prompted to enter their first and last names and their email address. -To add a new person, click **Add person** in the upper right corner. You’ll be prompted to enter their name and email address. +You can optionally add attributes to that user account, though you can add attributes to accounts at any time (as well as [via SSO](../people-and-groups/start.md#authentication)). Metabase uses attributes to create [data sandboxes](../permissions/data-sandboxes.md). -If you’ve already [configured Metabase to use email](../configuring-metabase/email.md), Metabase will send the new user an invite email. Otherwise, it’ll give you a temporary password that you’ll have to send to the person you’re inviting by hand. +Click **Create** to activate an account. An account becomes active once you click **Create**, even if the person never signs into the account. The account remains active until you [deactivate the account](#deactivating-an-account). If you're on a paid Metabase plan, all active accounts will count toward your user account total. If one person has more than one account, each account will count toward the total (see [how billing works](https://www.metabase.com/pricing/how-billing-works)). -### Deactivating an account +If you’ve already [configured Metabase to use email](../configuring-metabase/email.md), Metabase will send the person an email inviting them to log into Metabase. If you haven't yet setup email for your Metabase, Metabase will give you a temporary password that you’ll have to manually send to the person. + +## Editing an account + +You can edit someone's name and email address by clicking the three dots icon and choosing **Edit Details**. + +> Be careful: changing an account's email address _will change the address the person will use to log in to Metabase_. + +## Deactivating an account To deactivate someone's account, click on the three dots icon on the right of a person’s row and select **Deactivate** from the dropdown. Deactivating an account will mark it as inactive and prevent the user from logging in - but it _won’t_ delete that person's saved questions or dashboards. -![Remove a user](images/RemoveUser.png) +If you're using SSO, you should deactivate the account in Metabase as well as your IdP (that is, deactivation doesn't get applied from Metabase to your IdP, and vice versa). -### Reactivating an account +![Remove a user](images/RemoveUser.png) To reactivate a deactivated account, click the **Deactivated** radio button at the top of the people list to see the list of deactivated accounts. Click on the icon on the far right to reactivate that account, allowing them to log in to Metabase again. -### Deleting an account +## Deleting an account Metabase doesn't explicitly support account deletion. Instead, Metabase deactivates accounts so people can't log in to them, while it preserves any questions, models, dashboards, and other items created by those accounts. @@ -36,13 +44,9 @@ If you want to delete an account because the account information was set up inco 1. Change the name and email associated with the old account. 2. [Deactivate](#deactivating-an-account) the old account. -3. [Create a new account](#creating-accounts-for-your-team) with the person's correct information. - -### Editing an account +3. [Create a new account](#creating-an-account) with the person's correct information. -You can edit someone's name and email address by clicking the three dots icon and choosing **Edit Details**. Note: be careful when changing someone's email address, because _this will change the address they’ll use to log in to Metabase_. - -### Checking someone's auth method +## Checking someone's auth method Search for a person and look for an icon beside their name. @@ -51,13 +55,13 @@ Search for a person and look for an icon beside their name. Note that the type of user is set when the account is first created: if you create a user in Metabase, but that person then logs in via Google or some other form of SSO, the latter's icon will _not_ show up next to their name. -### Resetting someone’s password +## Resetting someone’s password If you've already [configured your email settings](../configuring-metabase/email.md), people can reset their passwords using the "forgot password" link on the login screen. If you haven't yet configured your email settings, they will see a message telling them to ask an admin to reset their password for them. To reset a password for someone, just click the three dots icon next to their account and choose **Reset Password**. If you haven’t [configured your email settings](../configuring-metabase/email.md) yet, you’ll be given a temporary password that you’ll have to share with that person. Otherwise, they’ll receive a password reset email. -### Resetting the admin password +## Resetting the admin password If you're using Metabase Cloud, [contact support](https://www.metabase.com/help-premium) to reset your admin password. @@ -84,7 +88,7 @@ If you're a Metabase admin and have access to the server console, you can get Me ``` 6. You should now see a page where you can input a new password for the admin account. -### Unsubscribe from all subscriptions / alerts +## Unsubscribe from all subscriptions / alerts This action will delete any dashboard subscriptions or alerts the person has created, and remove them as a recipient from any other subscriptions or alerts. @@ -117,31 +121,23 @@ The **All Users** group is another special one. Every Metabase user is always a It's important that your All Users group should never have _greater_ access for an item than a group for which you're trying to restrict access — otherwise the more permissive setting will win out. See [Setting permissions](../permissions/start.md). -### Managing groups - -#### Creating a group and adding people to it +## Creating a group -To create a group, go to **Admin settings** > **People** > **Groups**, and click the **Add a group** button. +Go to **Admin settings** > **People** > **Groups**, and click the **Add a group** button. We recommend creating groups that correspond to the teams your company or organization has, such as Human Resources, Engineering, Finance, and so on. By default, newly created groups don’t have access to anything. -Click into a group and then click `Add members` to add people to that group. Click on the X on the right side of a group member to remove them from that group. You can also add or remove people from groups from the People list using the dropdown in the Groups column. - -#### Removing a group - To remove a group, click the X icon to the right of a group in the list to remove it (remember, you can’t remove the special default groups). -#### Adding people to groups +## Adding people to groups -Adding people to groups allows you to assign +To add people to that group, click into a group and then click **Add members**. -- [Data access](../permissions/data.md), -- [Collection permissions](../permissions/collections.md), -- [Application permissions](../permissions/application.md). +To remove someone from that group, click on the **X** to the right of the group member. -To add someone to one or more groups, just click the Groups dropdown and click the checkboxes next to the group(s) you want to add the person to. You can also add people from the group's page. +You can also add or remove people from groups from the **People** list using the dropdown in the **Groups** column. -### Group managers +## Group managers {% include plans-blockquote.html feature="Group managers" %} @@ -156,7 +152,7 @@ Group managers can: Group managers are not admins, so their powers are limited. They cannot create new groups or invite new people to your Metabase. -#### Promoting/demoting group managers +## Promoting/demoting group managers To promote someone to become a group manager: @@ -164,12 +160,8 @@ To promote someone to become a group manager: 2. Select the group you want the person to manage. If the person isn't already in the group, you'll need to add that person to the group. 3. Find the person you want to promote, hover over their member type, and click the up arrow to promote them to group manager. If you want to demote them, click on the down arrow. -### Grouping strategies - -For guidance on which groups you should create for your Metabase, check out [Permissions strategies](https://www.metabase.com/learn/permissions/strategy). - ## Further reading -- [Configure Single Sign-On](./start.md). -- [Permissions overview](../permissions/start.md) -- [Learn permissions](https://www.metabase.com/learn/permissions/) +- [Configure Single Sign-On (SSO)](./start.md#authentication). +- [Permissions strategies](https://www.metabase.com/learn/permissions/strategy). +- [Multi-tenant permissions](https://www.metabase.com/learn/permissions/multi-tenant-permissions). diff --git a/docs/people-and-groups/start.md b/docs/people-and-groups/start.md index 2a28d1acedf2..acb4c28d9858 100644 --- a/docs/people-and-groups/start.md +++ b/docs/people-and-groups/start.md @@ -26,11 +26,7 @@ Tell Metabase how long it should wait before asking people to log in again. ## Authentication -Metabase offers several options for single sign-on (SS0) authentication. - -### Setting up Single Sign-on (SSO) - -We recommend that you set up [Single Sign-on][sso-def] for your Metabase installation. +Metabase offers several options for single sign-on (SSO) authentication. ### SSO for Metabase Open Source Edition diff --git a/docs/questions/images/Bookicon.png b/docs/questions/images/Bookicon.png deleted file mode 100644 index 912285bd57ce..000000000000 Binary files a/docs/questions/images/Bookicon.png and /dev/null differ diff --git a/docs/questions/images/DataReference.png b/docs/questions/images/DataReference.png deleted file mode 100644 index 91d41333a173..000000000000 Binary files a/docs/questions/images/DataReference.png and /dev/null differ diff --git a/docs/questions/native-editor/data-model-reference.md b/docs/questions/native-editor/data-model-reference.md deleted file mode 100644 index 4481cedb2913..000000000000 --- a/docs/questions/native-editor/data-model-reference.md +++ /dev/null @@ -1,25 +0,0 @@ ---- -title: Data reference -redirect_from: - - /docs/latest/users-guide/12-data-model-reference ---- - -## Data reference - -Sometimes when you're writing a SQL query, you might forget the exact names of different tables or columns, or which table contains what. That’s where the _data reference_ comes in handy. You can open the data reference panel from the SQL editor by clicking on the book icon in the top right corner of the editor when it's open. - -![The data reference button](../images/Bookicon.png) - -This panel lists all the databases you have access to and the tables and columns inside of them. - -![Data reference sidebar](../images/DataReference.png) - -Click on the table you would like to explore further to see a description of it and a list of all the columns it has. Each table or column will only contain a description if your admin wrote something in the Data Model section of the Admin Panel. - -If you click on a column you’re interested in, you’ll see a description of its contents, as well as a list of sample values for columns that don't have a huge number of distinct values. - -![Column detail](../images/data-reference-column-detail.png) - -### Foreign keys - -You can find a list of connections to other tables (i.e., [foreign key](https://www.metabase.com/glossary/foreign_key) relationships) under **Connected to these tables**. If there’s a column in the table you’re looking at that’s included in another table, Metabase will display this section in the sidebar. Note that connections are managed by your Metabase admin. If a foreign key relationship exists in your database, but not in your Metabase, your Metabase admin will need to update your [Metabase data model](../../data-modeling/metadata-editing.md). diff --git a/docs/questions/native-editor/referencing-saved-questions-in-queries.md b/docs/questions/native-editor/referencing-saved-questions-in-queries.md index 4b8faaadd3a2..341470179099 100644 --- a/docs/questions/native-editor/referencing-saved-questions-in-queries.md +++ b/docs/questions/native-editor/referencing-saved-questions-in-queries.md @@ -46,59 +46,55 @@ We'll select that question, and Metabase will update our code with the question' ``` SELECT count(*) -FROM {% raw %}{{#5}}{% endraw %} +FROM {% raw %}{{#5-gizmo-orders-in-2019}}{% endraw %} ``` This query returns the number of rows in our saved question. -## Saved question as a Common Table Expression (CTE) +## Model, table, or saved question as a Common Table Expression (CTE) -The same syntax can be used in [Common Table Expressions (CTEs)](https://en.wikipedia.org/wiki/Hierarchical_and_recursive_queries_in_SQL#Common_table_expression) (with SQL databases that support CTEs): +The same syntax can be used in [Common Table Expressions (CTEs)](https://www.metabase.com/learn/sql-questions/sql-cte) (with SQL databases that support CTEs): ``` -WITH 2019_gizmo_orders AS {% raw %}{{#5}}{% endraw %} +WITH gizmo_orders AS {% raw %}{{#5-gizmo-orders-in-2019}}{% endraw %} SELECT count(*) -FROM 2019_gizmo_orders +FROM gizmo_orders ``` -When this query is run, the `{% raw %}{{#5}}{% endraw %}` tag will be substituted with the SQL query of the referenced question, surrounded by parentheses. So it'll look like this under the hood: +When this query is run, the `{% raw %}{{#5-gizmo-orders-in-2019}}{% endraw %}` tag will be substituted with the SQL query of the referenced question, surrounded by parentheses. So it'll look like this under the hood: ``` -WITH 2019_gizmo_orders AS (SELECT * +WITH gizmo_orders AS (SELECT * FROM orders AS o INNER JOIN products AS p ON o.product_id = p.id WHERE p.category = 'Gizmo' AND o.created_at BETWEEN '2019-01-01' AND '2019-12-31') SELECT count(*) -FROM 2019_gizmo_orders +FROM gizmo_orders ``` -# How to find a question's ID +## Search for models and questions as you type -- Selecting a question from the variable sidebar in the SQL editor will automatically add the ID number to the variable in our query. -- You can also navigate to the question you'd like to reference and find its ID at the end of the URL in your browser's address bar, after `/question/`. E.g., for `https://metabase.example.com/question/12345`, the question's ID would be `12345`. +Use the typeahead search in the your variable to find your model or question. Type `{% raw %}{{#your search term }} {% endraw %}` and Metabase will display -## When and why to use saved questions as a data source +Selecting a question from the variable sidebar in the SQL editor will automatically add the ID number to the variable in our query. -- If you can't create a view in the database, since saved questions effectively act as views. If you can create a view, and you expect that you and others will frequently query the results, consider creating a materialized view. The results will be stored in the database (as opposed to computed each time), which will speed up query time. -- To simplify or standardize queries for people. If you have data split across multiple tables, you can perform those complicated joins once, and provide the results as a simplified "table" that people can query. +You can also navigate to the model or question you'd like to reference and find its ID in the URL in your browser's address bar, after `/model/` or `/question/`. E.g., for `https://metabase.example.com/model/12345-example-name`, the model's ID would be `12345`. -For other ways to standardize analytics, check out: +## Limitations and tradeoffs -- [Segments and Metrics](../../data-modeling/segments-and-metrics.md) -- [SQL Snippets](https://www.metabase.com/learn/building-analytics/sql-templates/sql-snippets.html) -- [SQL Snippets vs Saved Questions vs. Views](https://www.metabase.com/learn/building-analytics/sql-templates/organizing-sql.html) - -### Limitations and tradeoffs +- You can only reference a model or saved question in a query when working with a SQL database like PostgreSQL, MySQL, or SQL Server. +- The model or saved question you select has to be one that's based on the same database as the one you've currently selected in the native query editor. +- You cannot refer to variables in sub-queries. You only have access to the _results_ of the model or saved question, not the model or saved question's query. For example, if you have a saved question that uses a [field filter](https://www.metabase.com/learn/building-analytics/sql-templates/field-filters), you won't be able to reference that variable. If you need to change how the saved question has filtered the results, you'll need to update (or duplicate) that question and apply the filter. -- You can only reference a saved question in a query when working with a SQL database like PostgreSQL, MySQL, or SQL Server. -- The saved question you select has to be one that's based on the same database as the one you've currently selected in the native query editor. -- You cannot reference variables in sub-queries. You only have access to the _results_ of the saved question, not the saved question's query. For example, if you have a saved question that uses a [field filter](https://www.metabase.com/learn/building-analytics/sql-templates/field-filters), you won't be able to reference that variable. If you need to change how the saved question has filtered the results, you'll need to update (or duplicate) that question and apply the filter. +## Further reading -## Need help? - -If you're having trouble with your SQL query, go to the [SQL troubleshooting guide](../../troubleshooting-guide/sql.md). +- [Models](../../data-modeling/models.md) +- [SQL Snippets](https://www.metabase.com/learn/building-analytics/sql-templates/sql-snippets.html) +- [SQL Snippets vs Saved Questions vs. Views](https://www.metabase.com/learn/building-analytics/sql-templates/organizing-sql.html) +- [SQL troubleshooting guide](../../troubleshooting-guide/sql.md). +- [Segments and Metrics](../../data-modeling/segments-and-metrics.md) [cte]: https://www.metabase.com/learn/sql-questions/sql-cte diff --git a/docs/questions/native-editor/sql-parameters.md b/docs/questions/native-editor/sql-parameters.md index b21ef12bd18e..fd6b6cd3433f 100644 --- a/docs/questions/native-editor/sql-parameters.md +++ b/docs/questions/native-editor/sql-parameters.md @@ -253,6 +253,26 @@ An example for **BigQuery**, back ticks are needed, like `` FROM `dataset.table` For **Oracle** it would be `FROM "schema"."table"`. +### Include dependencies in your query + +Your main query should be aware of all the tables that your Field Filter variable is pointing to, otherwise you'll get a SQL syntax error. For example, let's say that your main query includes a field filter like this: + +``` +SELECT * +FROM ORDERS +WHERE {% raw %}{{ product_category }}{% endraw %} +``` + +Let's say the `{% raw %}{{ product_category }}{% endraw %}` variable refers to another question that uses the `Products` table. For the field filter to work, you'll need to include a join to `Products` in your main query. + +``` +SELECT * +FROM ORDERS o +JOIN PRODUCTS p +ON o.product_id = p.id +WHERE {% raw %}{{ product_category }}{% endraw %} +``` + ## Connecting a SQL question to a dashboard filter In order for a saved SQL/native question to be usable with a dashboard filter, the question must contain at least one variable. diff --git a/docs/questions/native-editor/writing-sql.md b/docs/questions/native-editor/writing-sql.md index 2d0680ee1111..c619120ba9f2 100644 --- a/docs/questions/native-editor/writing-sql.md +++ b/docs/questions/native-editor/writing-sql.md @@ -60,13 +60,11 @@ You can use [SQL snippets](sql-snippets.md) to save, reuse, and share SQL code a When you run a query from the SQL editor, Metabase sends the query to your database exactly as it is written. Any results or errors displayed in Metabase are the same as the results or errors that you would get if you ran the query directly against your database. If the SQL syntax of your query doesn’t match the SQL dialect used by your database, your database won’t be able to run the query. -## How Metabase executes SQL variables +## Question version history -When you run a query that includes a [variable][variable-gloss], the query will be executed by replacing the `{% raw %}{{ variable_name_or_id }}{% endraw %}` tag with the SQL query of the referenced question or model. +For questions, [dashboards](../../dashboards/start.md), and [models](../../data-modeling/models.md), Metabase keeps a version history for the previous fifteen versions of that item. -This means that your main query must be aware of all the tables that your variable is pointing to, otherwise you'll get a SQL syntax error. For example, if your main query uses the `Products` table, but your variable points to a query that uses the `Orders` table, you'll need to include a join to `Orders` in your main query. - -For an example, see the documentation on [Referencing models and saved questions in SQL queries](./referencing-saved-questions-in-queries.md). +See [History](../../exploration-and-organization/history.md). ## Learn more diff --git a/docs/questions/query-builder/expressions-list.md b/docs/questions/query-builder/expressions-list.md index f686425b7379..1b2a0cb5c84d 100644 --- a/docs/questions/query-builder/expressions-list.md +++ b/docs/questions/query-builder/expressions-list.md @@ -32,8 +32,10 @@ For an introduction to expressions, check out [Writing expressions in the notebo - [coalesce](./expressions/coalesce.md) - [concat](./expressions/concat.md) - [contains](#contains) - - [datetimeAdd](#datetimeadd) - - [datetimeSubtract](#datetimesubtract) + - [convertTimezone](./expressions/converttimezone.md) + - [datetimeAdd](./expressions/datetimeadd.md) + - [datetimeDiff](./expressions/datetimediff.md) + - [datetimeSubtract](./expressions/datetimesubtract.md) - [day](#day) - [endswith](#endswith) - [exp](#exp) @@ -47,6 +49,7 @@ For an introduction to expressions, check out [Writing expressions in the notebo - [log](#log) - [lower](#lower) - [minute](#minute) + - [month](#month) - [power](#power) - [quarter](#quarter) - [regexextract](./expressions/regexextract.md) @@ -59,6 +62,7 @@ For an introduction to expressions, check out [Writing expressions in the notebo - [substring](./expressions/substring.md) - [trim](#trim) - [upper](#upper) + - [week](#week) - [year](#year) - [Database limitations](#database-limitations) @@ -266,37 +270,49 @@ Example: `contains([Status], "Class")`. If `Status` were "Classified", the expre Related: [regexextract](#regexextract). -### datetimeAdd +### [convertTimezone](./expressions/converttimezone.md) + +Shifts a date or timestamp value into a specified time zone. + +Syntax: `convertTimezone(column, target, source)`. + +Example: `convertTimezone("2022-12-28T12:00:00", "Canada/Pacific", "Canada/Eastern")` would return the value `2022-12-28T09:00:00`, displayed as `December 28, 2022, 9:00 AM`. + +### [datetimeAdd](./expressions/datetimeadd.md) Adds some unit of time to a date or timestamp value. Syntax: `datetimeAdd(column, amount, unit)`. - - column: the column with your date or timestamp values. - - amount: The number of units to be added. - - units: "year", "quarter", "month", "day", "hour", "second", or "millisecond". +Example: `datetimeAdd("2021-03-25", 1, "month")` would return the value `2021-04-25`, displayed as `April 25, 2021`. + +Related: [between](#between), [datetimeSubtract](#datetimesubtract). + +### [datetimeDiff](./expressions/datetimediff.md) + +Returns the difference between two datetimes in some unit of time. For example, `datetimeDiff(d1, d2, "day") ` will return the number of days between `d1` and `d2`. -Example: `datetimeAdd("March 25, 2021, 12:52:37", 1, "month")` would return `April 25, 2021, 12:52:37`. +Syntax: `datetimeDiff(datetime1, datetime2, unit)`. -### datetimeSubtract +Example: `datetimeDiff("2022-02-01", "2022-03-01", "month")` would return `1`. + +### [datetimeSubtract](./expressions/datetimesubtract.md) Subtracts some unit of time from a date or timestamp value. Syntax: `datetimeSubtract(column, amount, unit)`. - - column: the column with your date or timestamp values. - - amount: The number of units to be subtracted. - - units: "year", "quarter", "month", "day", "hour", "second", or "millisecond". +Example: `datetimeSubtract("2021-03-25", 1, "month")` would return the value `2021-02-25`, displayed as `February 25, 2021`. -Example: `datetimeSubtract("March 25, 2021, 12:52:37", 1, "month")` would return `February 25, 2021, 12:52:37`. +Related: [between](#between), [datetimeAdd](#datetimeadd). ### day Takes a datetime and returns the day of the month as an integer. -Syntax: `day([datetime column)`. +Syntax: `day([datetime column])`. -Example: `day("March 25, 2021, 12:52:37")` would return the day as an integer, `25`. +Example: `day("2021-03-25T12:52:37")` would return the day as an integer, `25`. ### endswith @@ -332,9 +348,9 @@ Related: [ceil](#ceil), [round](#round). Takes a datetime and returns the hour as an integer (0-23). -Syntax: `hour([datetime column)`. +Syntax: `hour([datetime column])`. -Example: `hour("March 25, 2021, 12:52:37")` would return `12`. +Example: `hour("2021-03-25T12:52:37")` would return `12`. ### interval @@ -402,17 +418,17 @@ Related: [upper](#upper). Takes a datetime and returns the minute as an integer (0-59). -Syntax: `minute([datetime column)`. +Syntax: `minute([datetime column])`. -Example: `minute("March 25, 2021, 12:52:37")` would return `52`. +Example: `minute("2021-03-25T12:52:37")` would return `52`. ### month Takes a datetime and returns the month number (1-12) as an integer. -Syntax: `month([datetime column)`. +Syntax: `month([datetime column])`. -Example: `month("March 25, 2021, 12:52:37")` would return the month as an integer, `3`. +Example: `month("2021-03-25T12:52:37")` would return the month as an integer, `3`. ### power @@ -430,9 +446,9 @@ Related: [exp](#exp). Takes a datetime and returns the number of the quarter in a year (1-4) as an integer. -Syntax: `quarter([datetime column)`. +Syntax: `quarter([datetime column])`. -Example: `quarter("March 25, 2021, 12:52:37")` would return `1` for the first quarter. +Example: `quarter("2021-03-25T12:52:37")` would return `1` for the first quarter. ### [regexextract](./expressions/regexextract.md) @@ -478,7 +494,7 @@ Takes a datetime and returns the number of seconds in the minute (0-59) as an in Syntax: `second([datetime column)`. -Example: `second("March 25, 2021, 12:52:37")` would return the integer `37`. +Example: `second("2021-03-25T12:52:37")` would return the integer `37`. ### sqrt @@ -528,13 +544,27 @@ Syntax: `upper(text)`. Example: `upper([Status])`. If status were "hyper", `upper("hyper")` would return "HYPER". +### week + +Takes a datetime and returns the week as an integer. + +Syntax: `week(column, mode)`. + +Example: `week("2021-03-25T12:52:37")` would return the week as an integer, `12`. + +- column: the name of the column of the date or datetime value. +- mode: Optional. + - ISO: (default) Week 1 starts on the Monday before the first Thursday of January. + - US: Week 1 starts on Jan 1. All other weeks start on Sunday. + - Instance: Week 1 starts on Jan 1. All other weeks start on the day defined in your Metabase localization settings. + ### year Takes a datetime and returns the year as an integer. -Syntax: `year([datetime column)`. +Syntax: `year([datetime column])`. -Example: `year("March 25, 2021, 12:52:37")` would return the year 2021 as an integer, `2,021`. +Example: `year("2021-03-25T12:52:37")` would return the year 2021 as an integer, `2,021`. ## Database limitations diff --git a/docs/questions/query-builder/expressions/concat.md b/docs/questions/query-builder/expressions/concat.md index 865795e4a51a..3e8e98d0ad82 100644 --- a/docs/questions/query-builder/expressions/concat.md +++ b/docs/questions/query-builder/expressions/concat.md @@ -93,4 +93,4 @@ concat([City], ", ", [Country]) ## Further reading - [Custom expressions documentation](../expressions.md) -- [Custom expressions tutorial](https://www.metabase.com/learn/questions/) \ No newline at end of file +- [Custom expressions tutorial](https://www.metabase.com/learn/questions/custom-expressions) \ No newline at end of file diff --git a/docs/questions/query-builder/expressions/converttimezone.md b/docs/questions/query-builder/expressions/converttimezone.md new file mode 100644 index 000000000000..3d809842ca0e --- /dev/null +++ b/docs/questions/query-builder/expressions/converttimezone.md @@ -0,0 +1,249 @@ +--- +title: ConvertTimezone +--- + +# ConvertTimezone + +`convertTimezone` shifts a timestamp into a specified time zone by adding or subtracting the right interval from the timestamp. + +| Syntax | Example | +| --------------------------------------------------------------------- | --------------------------------------------------------------------------------- | +| `convertTimezone(column, target, source)` | `convertTimezone("2022-12-28T12:00:00", "Canada/Pacific", "Canada/Eastern")` | +| Shifts a timestamp from the source time zone to the target time zone. | Returns the value `2022-12-28T09:00:00`, displayed as `December 28, 2022 9:00 AM` | + +Timestamps and time zones are rather nasty to work with (it's easy to make mistakes, and difficult to catch them), so you should only try to use `convertTimezone` if the interpretation of your data is sensitive to time-based cutoffs. + +For example, if you're tracking user logins over time, you probably won't run your business differently if some logins get counted on Mondays instead of Tuesdays. However, if you're using Metabase to do something precise, like your taxes, you (and the government) will probably care a lot more about the difference between transactions that occurred on Dec 31 vs. Jan 1. + +## Parameters + +`column` can be any of: + +- The name of a timestamp column, +- a custom expression that returns a [timestamp](#accepted-data-types), or +- a string in the format `"YYYY-MM-DD` or `"YYYY-MM-DDTHH:MM:SS"`. + +`target`: + +- The name of the time zone you want to assign to your column. + +`source`: + +- The name of your column's current time zone. +- Required for columns or expressions with the data type `timestamp without time zone`. +- Optional for columns or expressions with the data type `timestamp with time zone`. +- For more info, see [Accepted data types](#accepted-data-types). + +We support [tz database](https://en.wikipedia.org/wiki/List_of_tz_database_time_zones) time zone names (such as "Canada/Eastern" instead of "EST"). + +## Creating custom report dates + +Let's say that you have some time series data that's stored in one or more time zones (**Source Time**). You want to create custom reporting dates for a team that lives in EST. + +| Source Time | Team Report Time (EST) | +| --------------------------- | --------------------------- | +| December 28, 2022, 10:00:00 | December 28, 2022, 07:00:00 | +| December 28, 2022, 21:00:00 | December 28, 2022, 19:00:00 | +| December 27, 2022, 08:00:00 | December 27, 2022, 05:00:00 | + +If **Source Time** is stored as a `timestamp with time zone` or a `timestamp with offset`, you only need to provide the `target` time zone: + +``` +convertTimezone([Source Time], 'EST') +``` + +If **Source Time** is stored as a `timestamp without time zone`, you _must_ provide the `source` time zone (which will depend on your database time zone): + +``` +convertTimezone([Source Time], 'EST', 'UTC') +``` + +It's usually a good idea to label `convertTimezone` columns with the name of the target time zone (or add the target time zone to the metadata of a model). We promise this will make your life easier when someone inevitably asks why the numbers don't match. + +If you're not getting the results that you expect: + +- Check if you have the right [source time zone](#choosing-a-source-time-zone). +- Ask your database admin about `timestamp with time zone` vs. `timestamp without time zone` (for more info, see [Accepted data types](#accepted-data-types)). + +### Choosing a source time zone + +When you're doing time zone conversions, make sure you know the source time zone that you're working with. Different columns (and even different rows) in the same table, question, or model can be in different "source" time zones. + +| Possible source time zone | Description | Example | +| ------------------------- | -------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------ | +| Client time zone | Time zone where an event happened. | A web analytics service might capture data in the local time zone of each person who visited your website. | +| Database time zone | Time zone metadata that's been added to timestamps in your database. | It's a common database practice to store all timestamps in UTC. | +| No time zone | Missing time zone metadata | Databases don't _require_ you to store timestamps with time zone metadata. | +| Metabase report time zone | Time zone that Metabase uses to _display_ timestamps. | Metabase can display dates and times in PST, even if the dates and times are stored as UTC in your database. | + +For example, say you have a table with one row for each person who visited your website. It's hard to tell, just from looking at `December 28, 2022, 12:00 PM`, whether the "raw" timestamp is: + +- stored using your database's time zone (usually UTC), +- stored without time zone metadata (for example, if the website visitor is in HKT, then the timestamp `December 28, 2022, 12:00 PM` might "implicitly" use Hong Kong time), +- _displayed_ in your Metabase report time zone. + +For more gory details, see [Limitations](#limitations). + +## Accepted data types + +| [Data type](https://www.metabase.com/learn/databases/data-types-overview#examples-of-data-types) | Works with `convertTimezone` | +| ------------------------------------------------------------------------------------------------ | ---------------------------- | +| String | ❌ | +| Number | ❌ | +| Timestamp | ✅ | +| Boolean | ❌ | +| JSON | ❌ | + +We use "timestamp" and "datetime" to talk about any temporal data type that's supported by Metabase. + +If your timestamps are stored as strings or numbers in your database, an admin can [cast them to timestamps](../../../data-modeling/metadata-editing.md#casting-to-a-specific-data-type) from the Data Model page. + +To use `convertTimezone` without running into errors or pesky undetectable mistakes, you should know that there are a few varieties of `timestamp` data types: + +| Data type | Description | Example | +| ----------------------------- | ----------------------------------------- | ---------------------------------------------------- | +| `timestamp with time zone` | Knows about location. | `2022-12-28T12:00:00 AT TIME ZONE 'America/Toronto'` | +| `timestamp with offset` | Knows about the time difference from UTC. | `2022-12-28T12:00:00-04:00` | +| `timestamp without time zone` | No time zone info. | `2022-12-28T12:00:00` | + +Note that the first part of the timestamp is in UTC (same thing as GMT). The time zone or offset tells you how much time to add or subtract for a given time zone. + +`convertTimezone` will work with all three types of timestamps, but the output of `convertTimezone` will always be a `timestamp without time zone`. + +## Limitations + +`convertTimezone` is currently unavailable for the following databases: + +- Amazon Athena +- Druid +- Google Analytics +- H2 +- MongoDB +- Presto +- SparkSQL +- SQLite + +### Notes on source time zones + +Metabase displays timestamps without time zone or offset information, which is why you have to be so careful about the [source time zone](#choosing-a-source-time-zone) when using `convertTimezone`. + +The Metabase report time zone only applies to `timestamp with time zone` or `timestamp with offset` data types. For example: + +| Raw timestamp in your database | Data type | Report time zone | Displayed as | +| ---------------------------------------- | ----------------------------- | ---------------- | ---------------------- | +| `2022-12-28T12:00:00 AT TIME ZONE 'CST'` | `timestamp with time zone` | 'Canada/Eastern' | Dec 28, 2022, 7:00 AM | +| `2022-12-28T12:00:00-06:00` | `timestamp with offset` | 'Canada/Eastern' | Dec 28, 2022, 7:00 AM | +| `2022-12-28T12:00:00` | `timestamp without time zone` | 'Canada/Eastern' | Dec 28, 2022, 12:00 AM | + +The Metabase report time zone will not apply to the output of a `convertTimezone` expression. For example: + +``` +convertTimezone("2022-12-28T12:00:00 AT TIME ZONE 'Canada/Central'", "Canada/Pacific", "Canada/Central") +``` + +will produce a raw `timestamp without time zone` + +``` +2022-12-28T04:00:00 +``` + +and displayed in Metabase as + +``` +Dec 28, 2022, 4:00 AM +``` + +If you use `convertTimezone` on a `timestamp without time zone`, make sure to use 'UTC' as the `source` time zone, otherwise the expression will shift your timestamp by the wrong amount. For example, if our `timestamp without time zone` is only "implied" to be in CST, we should use 'UTC' as the `source` parameter to get the same result as above. + +For example, if we choose 'CST' as the `source` time zone for a `timestamp without time zone`: + +``` +convertTimezone("2022-12-28T12:00:00", "Canada/Pacific", "Canada/Central") +``` + +we'll get the raw `timestamp without time zone` + +``` +2022-12-28T10:00:00 +``` + +displayed in Metabase as + +``` +Dec 28, 2022, 10:00 AM +``` + +## Related functions + +This section covers functions and formulas that work the same way as the Metabase `convertTimezone` expression, with notes on how to choose the best option for your use case. + +- [SQL](#sql) +- [Spreadsheets](#spreadsheets) +- [Python](#python) + +### SQL + +When you run a question using the [query builder](https://www.metabase.com/glossary/query_builder), Metabase will convert your graphical query settings (filters, summaries, etc.) into a query, and run that query against your database to get your results. + +If our [timestamp sample data](#creating-custom-report-dates) is a `timestamp without time zone` stored in a PostgreSQL database, the source time zone will always be the Postgres database time zone, so we can convert it directly using: + +```sql +SELECT source_time::TIMESTAMP AT TIME ZONE 'EST' AS team_report_time_est +``` + +which is the same as the `convertTimezone` expression _with_ a `source` parameter: + +``` +convertTimezone([Source Time], "Canada/Eastern", "UTC") +``` + +If `source_time` is a `timestamp with time zone` or `timestamp with offset` (for example, in a Snowflake database), then we don't need to specify a source time zone in SQL or in Metabase. + +```sql +SELECT convert_timezone('America/Toronto', source_time) AS team_report_time_est +``` + +is the same as + +``` +convertTimezone([Source Time], "Canada/Eastern") +``` + +Remember that the time zone names depend on your database. For example, Snowflake doesn't accept most time zone abbreviations (like EST). + +### Spreadsheets + +If our [timestamp sample data](#creating-custom-report-dates) is in a spreadsheet where "Source Time" is in column A, we can change it to EST by subtracting the hours explicitly: + +``` +A1 - TIME(5, 0, 0) +``` + +to get the same result as + +``` +convertTimezone([Client Time], "Canada/Eastern") +``` + +### Python + +If the [timestamp sample data](#creating-custom-report-dates) is stored in a `pandas` dataframe, you could convert the **Source Time** column to a `timestamp` object with time zone first(basically making a `timestamp without time zone` into a `timestamp with time zone`), then use `tz_convert` to change the time zone to EST: + +``` +df['Source Time (UTC)'] = pd.to_timestamp(df['Source Time'], utc=True) +df['Team Report Time (EST)'] = df['Source Time (UTC)'].dt.tz_convert(tz='Canada/Eastern') +``` + +to do the same thing as a nested `convertTimezone` expression + +``` +convertTimezone(convertTimezone([Source Time], "UTC"), "Canada/Eastern", "UTC") +``` + +## Further reading + +- [Custom expressions documentation](../expressions.md) +- [Custom expressions tutorial](https://www.metabase.com/learn/questions/custom-expressions) +- [Time series comparisons](https://www.metabase.com/learn/questions/time-series-comparisons) +- [How to compare one time period to another](https://www.metabase.com/learn/dashboards/compare-times) +- [Working with dates in SQL](https://www.metabase.com/learn/sql-questions/dates-in-sql) diff --git a/docs/questions/query-builder/expressions/datetimeadd.md b/docs/questions/query-builder/expressions/datetimeadd.md new file mode 100644 index 000000000000..6eecd449911d --- /dev/null +++ b/docs/questions/query-builder/expressions/datetimeadd.md @@ -0,0 +1,171 @@ +--- +title: DatetimeAdd +--- + +# DatetimeAdd + +`datetimeAdd` takes a datetime value and adds some unit of time to it. This function is useful when you're working with time series data that's marked by a "start" and an "end", such as sessions or subscriptions data. + +| Syntax | Example | +| ---------------------------------------------------------------------------------- | --------------------------------------- | +| `datetimeAdd(column, amount, unit)` | `datetimeAdd("2021-03-25", 1, "month")` | +| Takes a timestamp or date value and adds the specified number of time units to it. | `2021-04-25` | + +## Parameters + +`column` can be any of: + +- The name of a timestamp column, +- a custom expression that returns a [datetime](#accepted-data-types), or +- a string in the format `"YYYY-MM-DD"` or `"YYYY-MM-DDTHH:MM:SS"`(as shown in the example above). + +`unit` can be any of: + +- "year" +- "quarter" +- "month" +- "day" +- "hour" +- "second" +- "millisecond" + +`amount`: + +- A whole number or a decimal number. +- May be a negative number: `datetimeAdd("2021-03-25", -1, "month")` will return `2021-04-25`. + +## Calculating an end date + +Let's say you're a coffee connoisseur, and you want to keep track of the freshness of your beans: + +| Coffee | Opened On | Finish By | +| ---------------------- | ----------------- | ----------------- | +| DAK Honey Dude | October 31, 2022 | November 14, 2022 | +| NO6 Full City Espresso | November 7, 2022 | November 21, 2022 | +| Ghost Roaster Giakanja | November 27, 2022 | December 11, 2022 | + +Here, **Finish By** is a custom column with the expression: + +``` +datetimeAdd([Opened On], 14, 'day') +``` + +## Comparing a date to a window of time + +To check if a specific datetime falls between your start and end datetimes, use [`between`](../expressions-list.md#between). + +Unfortunately, Metabase doesn't currently support functions like `today`. If you want to check if today's date falls between **Opened On** and **Finish By** in the [Coffee example](#calculating-an-end-date): + +1. Ask your database admin if there's table in your database that stores dates for reporting (sometimes called a date dimension table). +2. Create a new question using the date dimension table, with a filter for "Today". +3. Turn the "Today" question into a [model](../../../data-modeling/models.md). +4. Create a [left join](../../query-builder/join.md) between **Coffee** and the "Today" model on `[Opened On] <= [Today]` and `[Finish By] >= [Today]`. + +The result should give you a **Today** column that's non-empty if today's date falls inside the coffee freshness window: + +| Coffee | Opened On | Finish By | Today | +| ---------------------- | ----------------- | ----------------- | ----------------- | +| DAK Honey Dude | October 31, 2022 | November 14, 2022 | November 11, 2022 | +| NO6 Full City Espresso | November 7, 2022 | November 21, 2022 | November 11, 2022 | +| Ghost Roaster Giakanja | November 27, 2022 | December 11, 2022 | | + +## Accepted data types + +| [Data type](https://www.metabase.com/learn/databases/data-types-overview#examples-of-data-types) | Works with `datetimeAdd` | +| ------------------------------------------------------------------------------------------------ | ------------------------ | +| String | ❌ | +| Number | ❌ | +| Timestamp | ✅ | +| Boolean | ❌ | +| JSON | ❌ | + +We use "timestamp" and "datetime" to talk about any temporal data type that's supported by Metabase. + +If your timestamps are stored as strings or numbers in your database, an admin can [cast them to timestamps](../../../data-modeling/metadata-editing.md#casting-to-a-specific-data-type) from the Data Model page. + +## Limitations + +If you're using MongoDB, `datetimeAdd` will only work on versions 5 and up. + +## Related functions + +This section covers functions and formulas that work the same way as the Metabase `datetimeAdd` expression, with notes on how to choose the best option for your use case. + +**[Metabase expressions](../expressions-list.md)** + +- [datetimeSubtract](#datetimesubtract) + +**Other tools** + +- [SQL](#sql) +- [Spreadsheets](#spreadsheets) +- [Python](#python) + +### datetimeSubtract + +`datetimeSubtract` and `datetimeAdd` are interchangeable, since you can use a negative number for `amount`. It's generally a good idea to avoid double negatives (such as subtracting a negative number). + +``` +datetimeSubtract([Opened On], -14, "day") +``` + +does the same thing as + +``` +datetimeAdd([Opened On], 14, "day") +``` + +### SQL + +When you run a question using the [query builder](https://www.metabase.com/glossary/query_builder), Metabase will convert your graphical query settings (filters, summaries, etc.) into a query, and run that query against your database to get your results. + +If our [coffee sample data](#calculating-an-end-date) is stored in a PostgreSQL database: + +```sql +SELECT opened_on + INTERVAL '14 days' AS finish_by +FROM coffee +``` + +is equivalent to the Metabase `datetimeAdd` expression: + +``` +datetimeAdd([Opened On], 14, "day") +``` + +### Spreadsheets + +If our [coffee sample data](#calculating-an-end-date) is in a spreadsheet where "Opened On" is in column A with a date format, the spreadsheet function + +``` +A:A + 14 +``` + +produces the same result as + +``` +datetimeAdd([Opened On], 14, "day") +``` + +Most spreadsheet tools require use different functions for different time units (for example, you'd use a different function to add "months" to a date). `datetimeAdd` makes it easy for you to convert all of those functions to a single consistent syntax. + +### Python + +Assuming the [coffee sample data](#calculating-an-end-date) is in a `pandas` dataframe column called `df`, you can import the `datetime` module and use the `timedelta` function: + +``` +df['Finish By'] = df['Opened On'] + datetime.timedelta(days=14) +``` + +is equivalent to + +``` +datetimeAdd([Opened On], 14, "day") +``` + +## Further reading + +- [Custom expressions documentation](../expressions.md) +- [Custom expressions tutorial](https://www.metabase.com/learn/questions/custom-expressions) +- [Time series comparisons](https://www.metabase.com/learn/questions/time-series-comparisons) +- [How to compare one time period to another](https://www.metabase.com/learn/dashboards/compare-times) +- [Working with dates in SQL](https://www.metabase.com/learn/sql-questions/dates-in-sql) diff --git a/docs/questions/query-builder/expressions/datetimediff.md b/docs/questions/query-builder/expressions/datetimediff.md new file mode 100644 index 000000000000..395697e1b9f2 --- /dev/null +++ b/docs/questions/query-builder/expressions/datetimediff.md @@ -0,0 +1,164 @@ +--- +title: DatetimeDiff +--- + +# DatetimeDiff + +`datetimeDiff` gets the amount of time between two datetime values, using the specified unit of time. Note that the difference is calculated in _whole_ units (see the example below). + +| Syntax | Example | +| -------------------------------------------------------------------------------------------------------- | --------------------------------------------------- | +| `datetimeDiff(datetime1, datetime2, unit)` | `datetimeDiff("2022-02-01", "2022-03-01", "month")` | +| Gets the difference between two datetimes (datetime2 minus datetime 1) using the specified unit of time. | `1` | + +## Parameters + +`datetime1` and `datetime2` can be: + +- The name of a timestamp column, +- a custom expression that returns a [datetime](#accepted-data-types), or +- a string in the format `"YYYY-MM-DD"` or `"YYYY-MM-DDTHH:MM:SS"` (as shown in the example above). + +`unit` can be any of: + +- "year" +- "month" +- "day" +- "hour" +- "second" +- "millisecond" + +## Limitations + +`datetimeDiff` is currently unavailable for the following databases: + +- Amazon Athena +- Druid +- Google Analytics +- H2 +- MongoDB +- Oracle +- Presto +- Redshift +- SparkSQL +- SQLite +- SQL Server +- Vertica + +## Calculating age + +Let's say you're a cheesemaker, and you want to keep track of your ripening process: + +| Cheese | Aging Start | Aging End | Mature Age (Months) | +| ------------- | ---------------- | ---------------- | ------------------- | +| Provolone | January 19, 2022 | March 17, 2022 | 1 | +| Feta | January 25, 2022 | May 3, 2022 | 3 | +| Monterey Jack | January 27, 2022 | October 11, 2022 | 8 | + +**Mature Age (Months)** is a custom column with the expression: + +``` +datetimeDiff([Aging Start], [Aging End], "month") +``` + +## Calculating current age + +Metabase doesn't currently support datetime functions like `today`. To calculate the _current_ age in our [cheese example](#calculating-age): + +1. Ask your database admin if there's table in your database that stores dates for reporting (sometimes called a date dimension table). +2. Create a new question using the date dimension table, with a filter for "Today". +3. Turn the "Today" question into a [model](../../../data-modeling/models.md). +4. Create a [left join](../../query-builder/join.md) between **Cheese** and the "Today" model on `[Aging Start] <= [Today]`. + +The result should give you a **Today** column that's non-empty if today's date is on or after the **Aging Start** date. + +| Cheese | Aging Start | Aging End | Mature Age (Months) | Today | Current Age (Months) | +| ------------- | ---------------- | ---------------- | ------------------- | ------------------ | -------------------- | +| Provolone | January 19, 2022 | March 17, 2022 | 1 | September 19, 2022 | 8 | +| Feta | January 25, 2022 | May 3, 2022 | 3 | September 19, 2022 | 7 | +| Monterey Jack | January 27, 2022 | October 11, 2022 | 8 | September 19, 2022 | 7 | + +Then, you can calculate **Current Age (Months)** like this: + +``` +datetimeDiff([Aging Start], [Today], "month") +``` + +## Accepted data types + +| [Data type](https://www.metabase.com/learn/databases/data-types-overview#examples-of-data-types) | Works with `datetimeDiff` | +| ------------------------------------------------------------------------------------------------ | ------------------------- | +| String | ❌ | +| Number | ❌ | +| Timestamp | ✅ | +| Boolean | ❌ | +| JSON | ❌ | + +We use "timestamp" and "datetime" to talk about any temporal data type that's supported by Metabase. + +If your timestamps are stored as strings or numbers in your database, an admin can [cast them to timestamps](../../../data-modeling/metadata-editing.md#casting-to-a-specific-data-type) from the Data Model page. + +## Related functions + +This section covers functions and formulas that work the same way as the Metabase `datetimeDiff` expression, with notes on how to choose the best option for your use case. + +- [SQL](#sql) +- [Spreadsheets](#spreadsheets) +- [Python](#python) + +### SQL + +When you run a question using the [query builder](https://www.metabase.com/glossary/query_builder), Metabase will convert your graphical query settings (filters, summaries, etc.) into a query, and run that query against your database to get your results. + +If our [cheese sample data](#calculating-age) is stored in a PostgreSQL database: + +```sql +SELECT DATE_PART('month', AGE(aging_end, aging_start)) AS mature_age_months +FROM cheese +``` + +is equivalent to the Metabase `datetimeDiff` expression: + +``` +datetimeDiff([Aging Start], [Aging End], "month") +``` + +Some databases, such as Snowflake and BigQuery, support functions like `DATEDIFF` or `DATE_DIFF`. For more info, check out our list of [common SQL reference guides](https://www.metabase.com/learn/debugging-sql/sql-syntax#common-sql-reference-guides). + +### Spreadsheets + +If our [cheese sample data](#calculating-age) is in a spreadsheet where "Aging Start" is in column B and "Aging End" is in column C: + +``` +DATEDIF(B1, C1, "M") +``` + +produces the same result as + +``` +datetimeDiff([Aging Start], [Aging End], "month") +``` + +Yes, `DATEDIF` looks a bit wrong, but the spreadsheet function really is `DATEDIF()` with one "f", not `DATEDIFF()`. + +### Python + +Assuming the [cheese sample data](#calculating-age) is in a `pandas` dataframe column called `df`, you can subtract the dates directly and use `numpy`'s `timedelta64` to convert the difference to months: + +``` +df['Mature Age (Months)'] = (df['Aging End'] - df['Aging Start']) / np.timedelta64(1, 'M') +``` + +is equivalent to + +``` +datetimeDiff([Aging Start], [Aging End], "month") +``` + +## Further reading + +- [Custom expressions documentation](../expressions.md) +- [Custom expressions tutorial](https://www.metabase.com/learn/questions/custom-expressions) +- [Time series comparisons](https://www.metabase.com/learn/questions/time-series-comparisons) +- [How to compare one time period to another](https://www.metabase.com/learn/dashboards/compare-times) +- [Working with dates in SQL](https://www.metabase.com/learn/sql-questions/dates-in-sql) diff --git a/docs/questions/query-builder/expressions/datetimesubtract.md b/docs/questions/query-builder/expressions/datetimesubtract.md new file mode 100644 index 000000000000..8ef899238711 --- /dev/null +++ b/docs/questions/query-builder/expressions/datetimesubtract.md @@ -0,0 +1,168 @@ +--- +title: DatetimeSubtract +--- + +# DatetimeSubtract + +`datetimeSubtract` takes a datetime value and subtracts some unit of time from it. You might want to use this function when working with time series data that's marked by a "start" and an "end", such as sessions or subscriptions data. + +| Syntax | Example | +|-------------------------------------------------------------------------------------------|---------------------------------------------| +| `datetimeSubtract(column, amount, unit)` | `datetimeSubtract("2021-03-25", 1, "month")`| +| Takes a timestamp or date value and subtracts the specified number of time units from it. | `2021-02-25` | + +## Parameters + +`column` can be any of: +- The name of a timestamp column, +- a custom expression that returns a [datetime](#accepted-data-types), or +- a string in the format `"YYYY-MM-DD"` or `"YYYY-MM-DDTHH:MM:SS"` (as shown in the example above). + +`unit` can be any of: +- "year" +- "quarter" +- "month" +- "day" +- "hour" +- "second" +- "millisecond" + +`amount`: +- A whole number or a decimal number. +- May be a negative number: `datetimeSubtract("2021-03-25", -1, "month")` will return `2021-04-25`. + +## Calculating a start date + +Let's say you're planning a fun night out. You know it takes 30 minutes to get from place to place, and you need to figure out what time you have to leave to get to each of your reservations: + +| Event | Arrive By | Depart At | +|---------|----------------------------|-----------------------------| +| Drinks | November 12, 2022 6:30 PM | November 12, 2022 6:00 PM | +| Dinner | November 12, 2022 8:00 PM | November 12, 2022 7:30 PM | +| Dancing | November 13, 2022 12:00 AM | November 12, 2022 11:30 PM | + +Here, **Depart At** is a custom column with the expression: + +``` +datetimeSubtract([Arrive By], 30, "minute") +``` + +## Comparing a date to a window of time + +To check if an existing datetime falls between your start and end datetimes, use [`between`](../expressions-list.md#between). + +Unfortunately, Metabase doesn't currently support datetime functions like `today`. What if you want to check if today's date falls between **Arrive By** and **Depart At** in our [events example](#calculating-a-start-date)? + +1. Ask your database admin if there's table in your database that stores datetimes for reporting (sometimes called a date dimension table). +2. Create a new question using the date dimension table, with a filter for "Today". +3. Turn the "Today" question into a [model](../../../data-modeling/models.md). +4. Create a [left join](../../query-builder/join.md) between **Events** and the "Today" model on `[Arrive By] <= [Today]` and `[Depart At] >= [Today]`. + +The result should give you an **Today** column that's non-empty for events that are happening while the night is still young: + +| Event | Arrive By | Depart At | Today | +|---------|----------------------------|-----------------------------|-----------------------------| +| Drinks | November 12, 2022 6:30 PM | November 12, 2022 6:00 PM | November 12, 2022 12:00 AM | +| Dinner | November 12, 2022 8:00 PM | November 12, 2022 7:30 PM | November 12, 2022 12:00 AM | +| Dancing | November 13, 2022 12:00 AM | November 12, 2022 11:30 PM | | + +## Accepted data types + +| [Data type](https://www.metabase.com/learn/databases/data-types-overview#examples-of-data-types) | Works with `datetimeSubtract` | +| ----------------------- | -------------------- | +| String | ❌ | +| Number | ❌ | +| Timestamp | ✅ | +| Boolean | ❌ | +| JSON | ❌ | + +We use "timestamp" and "datetime" to talk about any temporal data type that's supported by Metabase. + +If your timestamps are stored as strings or numbers in your database, an admin can [cast them to timestamps](../../../data-modeling/metadata-editing.md#casting-to-a-specific-data-type) from the Data Model page. + +## Limitations + +If you're using MongoDB, `datetimeSubtract` will only work on versions 5 and up. + +## Related functions + +This section covers functions and formulas that work the same way as the Metabase `datetimeSubtract` expression, with notes on how to choose the best option for your use case. + +**[Metabase expressions](../expressions-list.md)** + +- [datetimeAdd](#datetimeadd) + +**Other tools** + +- [SQL](#sql) +- [Spreadsheets](#spreadsheets) +- [Python](#python) + +### datetimeAdd + +`datetimeSubtract` and `datetimeAdd` are interchangeable, since you can use a negative number for `amount`. We could use either expression for our [events example](#calculating-a-start-date), but you should try to avoid "double negatives" (such as subtracting a negative number). + +``` +datetimeAdd([Arrive By], -30, "minute") +``` + +does the same thing as + +``` +datetimeSubtract([Arrive By], 30, "minute") +``` + +### SQL + +When you run a question using the [query builder](https://www.metabase.com/glossary/query_builder), Metabase will convert your graphical query settings (filters, summaries, etc.) into a query, and run that query against your database to get your results. + +If our [events sample data](#calculating-a-start-date) is stored in a PostgreSQL database: + +```sql +SELECT arrive_by - INTERVAL '30 minutes' AS depart_at +FROM events +``` + +is equivalent to the Metabase `datetimeSubtract` expression: + +``` +datetimeSubtract([Arrive By], 30, "minute") +``` + +### Spreadsheets + +Assuming the [events sample data](#calculating-a-start-date) is in a spreadsheet where "Arrive By" is in column A with a datetime format, the spreadsheet function + +``` +A:A - 30/(60*24) +``` + +produces the same result as + +``` +datetimeSubtract([Arrive By], 30, "minute") +``` + +Most spreadsheets require you to use different calculations for different time units (for example, you'd need to use a different calculation to subtract "days" from a date). `datetimeSubtract` makes it easy for you to convert all of those functions to a single consistent syntax. + +### Python + +If our [events sample data](#calculating-a-start-date) is in a `pandas` dataframe column called `df`, you can import the `datetime` module and use the `timedelta` function: + +``` +df['Depart At'] = df['Arrive By'] - datetime.timedelta(minutes=30) +``` + +is equivalent to + +``` +datetimeSubtract([Arrive By], 30, "minute") +``` + +## Further reading + +- [Custom expressions documentation](../expressions.md) +- [Custom expressions tutorial](https://www.metabase.com/learn/questions/custom-expressions) +- [Time series comparisons](https://www.metabase.com/learn/questions/time-series-comparisons) +- [How to compare one time period to another](https://www.metabase.com/learn/dashboards/compare-times) +- [Working with dates in SQL](https://www.metabase.com/learn/sql-questions/dates-in-sql) \ No newline at end of file diff --git a/docs/questions/query-builder/expressions/regexextract.md b/docs/questions/query-builder/expressions/regexextract.md index bc4f11b0d94e..bfab1d90d135 100644 --- a/docs/questions/query-builder/expressions/regexextract.md +++ b/docs/questions/query-builder/expressions/regexextract.md @@ -37,7 +37,7 @@ regexextract([URL], "^[^?#]+\?utm_campaign=(.*)") Here, the regex pattern [`^[^?#]+\?` matches all valid URL strings](https://www.oreilly.com/library/view/regular-expressions-cookbook/9780596802837/ch07s13.html). You can replace `utm_campaign=` with whatever query parameter you like. At the end of the regex pattern, the [capturing group](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Guide/Regular_Expressions/Groups_and_Backreferences) `(.*)` gets all of the characters that appear after the query parameter `utm_campaign=`. -Now, you can use **Campaign Name** in places where you need clean labels, such as [filter dropdown menus](../../../dashboards/filters.md#choosing-between-a-dropdown-or-autocomplete-for-your-filter), [charts](../../sharing/visualizing-results.md), and [embedding parameters](../../../embedding/signed-embedding-parameters.md). +Now, you can use **Campaign Name** in places where you need clean labels, such as [filter dropdown menus](../../../dashboards/filters.md#creating-a-dropdown-filter), [charts](../../sharing/visualizing-results.md), and [embedding parameters](../../../embedding/signed-embedding-parameters.md). ## Accepted data types @@ -137,4 +137,4 @@ regexextract([URL], "^[^?#]+\?utm_campaign=(.*)") ## Further reading - [Custom expressions documentation](../expressions.md) -- [Custom expressions tutorial](https://www.metabase.com/learn/questions/) +- [Custom expressions tutorial](https://www.metabase.com/learn/questions/custom-expressions) diff --git a/docs/questions/query-builder/expressions/substring.md b/docs/questions/query-builder/expressions/substring.md index efb5f43e8d67..be8b2ce135b5 100644 --- a/docs/questions/query-builder/expressions/substring.md +++ b/docs/questions/query-builder/expressions/substring.md @@ -149,4 +149,4 @@ substring([Mission ID], 9, 3) ## Further reading - [Custom expressions documentation](../expressions.md) -- [Custom expressions tutorial](https://www.metabase.com/learn/questions/) +- [Custom expressions tutorial](https://www.metabase.com/learn/questions/custom-expressions) diff --git a/docs/questions/query-builder/introduction.md b/docs/questions/query-builder/introduction.md index edbbd0441bf9..242e93274835 100644 --- a/docs/questions/query-builder/introduction.md +++ b/docs/questions/query-builder/introduction.md @@ -6,7 +6,7 @@ redirect_from: # Asking questions -Metabase's two core concepts are questions and their corresponding answers. Everything else is based around questions and answers. To ask a question in Metabase, click the **+ New** button in the upper right of the main navigation bar, and select either: +Metabase's two core concepts are questions and their corresponding answers. To ask a question in Metabase, click the **+ New** button in the upper right of the main navigation bar, and select either: - Question - [SQL query](../native-editor/writing-sql.md) @@ -19,7 +19,7 @@ From the **+ New** dropdown, select **Question**, then pick your starting data: You can start a question from: -- **A model**. A [model][model] is a special kind of saved question meant to be used as a good starting point for questions. Sometimes these are called derived tables, as they usually pull together data from multiple raw tables. +- **A model**. A [model](../../data-modeling/models.md) is a special kind of saved question meant to be used as a good starting point for questions. Sometimes these are called derived tables, as they usually pull together data from multiple raw tables. - **Raw data**. You'll need to specify the database and the table in that database as the starting point for your question. - A **saved question**. You can use the results of any question as the starting point for a new question. @@ -61,13 +61,7 @@ Filtering just means narrowing things down based on certain criteria. You're pro ![Filtering](../images/filter-step.png) -When you add a filter step, you can select one or more columns to filter on. Depending on the type of column you pick, you'll get different options, like a calendar for date columns. - -Broadly speaking, there are three types of columns, each with their own set of filtering options: - -- **Numeric columns** let you add filters to only include rows in your table where this number is between two specific values, or is greater or less than a specific value, or is exactly equal to something. -- **Text or category columns** let you specify that you only want to include data where this column is or isn't a specific option, or you can exclude empty cells in that column. - - **Date** columns give you a lot of options to filter by specific date ranges, relative date ranges, and more. +When you add a filter step, you can select one or more columns to filter on. Depending on the [data type](https://www.metabase.com/learn/databases/data-types-overview) of the column you pick, you'll get different [filter types](#filter-types), like a calendar for date columns. You can add subsequent filter steps after each summarize step. This lets you do things like summarize by the count of rows per month, and then add a filter on the `count` column to only include rows where the count is greater than 100. (This is basically like a SQL `HAVING` clause.) @@ -77,6 +71,14 @@ Once you're happy with your filter, click **Add filter**, and visualize your res If you want to edit your filter, just click the little purple filter at the top of the screen. If you click on the X, you'll remove your filter. You can add as many filters as you need. +## Filter types + +Broadly speaking, there are three types of columns, each with their own set of filtering options: + +- **Numeric columns** let you add filters to only include rows in your table where this number is between two specific values, or is greater or less than a specific value, or is exactly equal to something. +- **Text or category columns** let you specify that you only want to include data where this column is or isn't a specific option, or you can exclude empty cells in that column. +- **Date** columns give you a lot of options to filter by specific date ranges, relative date ranges, and more. + ## Filter modal When viewing a table or chart, clicking on the **Filter** will bring up the filter modal: @@ -217,9 +219,14 @@ Feel free to play around with any saved question, as you won't have any effect o If you find yourself using the same saved question as a starting point for multiple questions, you may want to turn it into a [Model][model] to let others know it's a good starting place. +## Question version history + +For questions, [dashboards](../../dashboards/start.md), and [models](../../data-modeling/models.md), Metabase keeps a version history for the previous fifteen versions of that item. + +See [History](../../exploration-and-organization/history.md). + ## Further reading - [Visualize results](../sharing/visualizing-results.md). +- [Sharing answers](../sharing/answers.md). - [Asking questions](https://www.metabase.com/learn/questions) - -[model]: ../../data-modeling/models.md diff --git a/docs/questions/sharing/answers.md b/docs/questions/sharing/answers.md index 845aa8afc1f7..46b4f1dd2b27 100644 --- a/docs/questions/sharing/answers.md +++ b/docs/questions/sharing/answers.md @@ -36,7 +36,7 @@ Once you save your question, a down arrow will appear to the right of the questi - **Archive** (Folder with down arrow). See [Archiving items][archiving-items]. - **Bookmark** Save the question as a favorite, which will show up in the bookmarks section of your navigation sidebar. See [Bookmarks](../../exploration-and-organization/exploration.md#bookmarks). -### Caching results +## Caching results {% include plans-blockquote.html feature="Question-specific caching" %} @@ -46,26 +46,6 @@ Administrators can set global caching controls, but if you're using a paid versi Admins can still set global caching, but setting a cache duration on a specific question will override that global setting–useful for when a particular question has a different natural cadence. -### Question moderation - -{% include plans-blockquote.html feature="Question moderation" %} - -Administrators can **Verify** a question by clicking on the **Verify checkmark** in the **Moderation** section of the **Question detail sidebar**. Verifying a question is a simple way for an administrator to signal that they've reviewed the question and deemed it to be trustworthy. That is: the question is filtering the right columns, or summarizing the right metrics, and querying records from the right tables. - -Once verified, the question will have a verified icon next to the question's title. - -![Verified icon](../images/verified-icon.png) - -Verified questions are also more likely to show up higher in search suggestions and search results. - -If someone modifies a verified question, the question will lose its verified status, and an administrator will need to review and verify the question again to restore its verified status. - -### Question and model histories - -You can see the history of a question or [model][model], including edits and verifications, in the **History** section of the **Question detail sidebar**. - -Below each edit entry in the timeline, you can click on **Revert** to reinstate the question at the time of the edit. - ## Sharing questions with public links If your Metabase administrator has enabled [public sharing](../../questions/sharing/public-links.md) on a saved question or dashboard, you can go to that question or dashboard and click on the sharing icon to find its public links. Public links can be viewed by anyone, even if they don't have access to Metabase. You can also use the public embedding code to embed your question or dashboard in a simple web page or blog post. @@ -79,7 +59,7 @@ To share a question, click on the arrow pointing up and to the right in the bott You can set up questions to run periodically and notify you if the results are interesting. Check out [Alerts][alerts]. [alerts]: ./alerts.md -[archiving-items]: ../../exploration-and-organization/collections.md#archiving-items +[archiving-items]: ../../exploration-and-organization/history.md#archiving-items [caching]: ../../configuring-metabase/caching.md [collections]: ../../exploration-and-organization/collections.md [collection-permissions]: ../../permissions/collections.md diff --git a/docs/questions/sharing/public-links.md b/docs/questions/sharing/public-links.md index c815c80942ba..0ed3c28ecee0 100644 --- a/docs/questions/sharing/public-links.md +++ b/docs/questions/sharing/public-links.md @@ -1,82 +1,136 @@ --- -title: Public links +title: Public sharing redirect_from: - /docs/latest/administration-guide/12-public-links - /docs/latest/embedding/12-public-links + - /docs/latest/questions/sharing/public-links --- -# Public links +# Public sharing -Sometimes you'll want to share a dashboard or question you've saved with someone that isn't a part of your organization or company, or someone who doesn't need access to your full Metabase instance. Metabase lets administrators create public links and [public embeds](#public-embed) to let you do just that. +Sometimes you'll want to share a dashboard or question with someone who isn't a part of your organization or company. To share your work with people who don't need access to your full Metabase instance, you can create public links and public embeds. -## Turning public links on +## Enable public sharing in Metabase ![Enable public sharing](../images/enable-public-sharing.png) -First things first, you'll need to go to the Admin Panel and enable public sharing. In the future, you'll see dashboards and questions you've shared listed here, and you'll be able to revoke any public links that you no longer want to be used. + +In order to share Metabase items like questions and dashboards via a public link or an embedded iframe, an admin needs to enable public sharing for that Metabase instance by going to **Settings gear icon** > **Admin settings** > **Public sharing**. + +Once toggled on, the **Public sharing** section will display Metabase questions and dashboards with active public links. To deactivate a public link, click **Revoke link**. ## Enable sharing on your saved question or dashboard +Once [public sharing](#enable-sharing-on-your-saved-question-or-dashboard) is enabled for your Metabase, you'll find the **Sharing and embedding icon** on saved questions and dashboards (it looks like a box with an arrow pointing to the upper right). + +You can find the **Sharing and embedding icon** icon at the bottom right corner of a question, or the top right corner of a dashboard. + +To enable public sharing on a question or dashboard, click on the **Sharing and embedding icon** icon to bring up the sharing settings modal, then click on the toggle. + ![Enable sharing](../images/enable-links.png) -Next, exit the Admin Panel and go to question that you want to share, then click on the `Sharing and Embedding` icon in the bottom-right of the screen (it looks like an arrow pointing up and to the right). Then click on the toggle to enable public sharing for this question. +For more information about the option to **Embed this item in an application**, see the docs on [signed embedding](../../embedding/signed-embedding.md). -In the case of a dashboard, the button is located on the top right of the page. +## Public links -## Copy, paste, and share! +Once you've [enabled sharing on your question or dashboard](#enable-sharing-on-your-saved-question-or-dashboard), you can copy and share the public link URL with whomever you please. The public link URL will display static (view-only) results of your question or dashboard, so visitors won't be able to drill-down into the underlying data on their own. -Copy and share the public link URL with whomever you please. The public link URL will display static results of your question or dashboard, so visitors won't be able to drill-down into the underlying data on their own. +If you want to create a drill-down pathway on your question or dashboard, you can set up a [custom destination](../../dashboards/interactive.md) that goes to the public link of another question or dashboard. -However, public URLs preserve [custom click behavior](../../dashboards/interactive.md). If you like, you can share specific drill-down views by linking to other questions or dashboards. +### Public link to export question results in CSV, XLSX, JSON -## Public exports for question results in CSV, XLSX, JSON +The export option is only available for questions, not dashboards. -To create a public link to download the results of a question: +To create a public link that people can use to download the results of a question: -- Click on the **Sharing and embedding** icon for the question, -- Enable sharing, -- Then, below the **Public link** option, click on the format you want (CSV, XLSX, or JSON). Metabase will update the link based on your selection. -- Copy the link and test it out to confirm that the link downloads the expected format. +1. Click on the **Sharing and embedding** icon for the question. +2. Click the toggle to [enable sharing](#enable-sharing-on-your-saved-question-or-dashboard). +3. Click on the file format you want (below the **Public link** URL). +4. Open the public link in a new tab to test the download. ![Public export](../images/public-export.png) -This public link export option is only available for questions, not dashboards. +## Public embeds + +If you want to embed your question or dashboard in a simple web page or blog post: + +1. Click on the **Sharing and embedding** icon for your question or dashboard. +2. Click the toggle to [enable sharing](#enable-sharing-on-your-saved-question-or-dashboard). +3. Copy the **Public embed** iframe snippet. +4. Paste the iframe snippet in your destination of choice. -## Public embed +To customize the appearance of your question or dashboard, you can update the link in the `src` attribute with [public embed parameters](#public-embed-parameters). -If you want to embed the link to your dashboard or question in a simple web page or blog post, copy and paste the iframe snippet to your destination of choice. +## Public embed parameters -## Assigning values to filters or hiding them via the URL +To apply appearance or filter settings to your public embed, you can add parameters to the end of the link in your iframe's `src` attribute. -This is a bit more advanced, but if you're embedding a dashboard or question in an iframe and it has one or more filter widgets on it, you can give those filters values and even hide one or more filters by adding some options to the end of the URL. (You could also do this when just sharing a link, but note that if you do that, the person you're sharing the link with could of course directly edit the URL to change the filters' values, or to change which filters are hidden or not.) +Note that it's possible to find the public link URL behind a public embed. If someone gets access to the public link URL, they can remove the parameters from the URL to view the original question or dashboard (that is, without any appearance or filter settings). -Here's an example where we have a dashboard that has a couple filters on it, one of which is called "ID." We can give this filter a value of 7 and simultaneously prevent the filter widget from showing up by constructing our URL like this: +If you'd like to create a secure embed that prevents people from changing filter names or values, check out [signed embedding](../../embedding/signed-embedding.md). + +### Appearance parameters + +To toggle appearance settings, add _hash_ parameters to the end of the public link in your iframe's `src` attribute. + +For example, to embed a dashboard with a dark theme, original title, and no border: ``` -/dashboard/42?id=7#hide_parameters=id +/dashboard/42#theme=night&titled=true&bordered=false +``` + +| Parameter name | Possible values | +| ----------------------- | ------------------------------------------------ | +| bordered | true, false | +| titled | true, false | +| theme | null, transparent, night | +| hide_parameters | true, false | +| font¹ | [font name](../../configuring-metabase/fonts.md) | +| hide_download_button² | true, false | + +¹ Available on [paid plans](https://www.metabase.com/pricing). + +² Available on [paid plans](https://www.metabase.com/pricing) and hides the download button on questions only (not dashboards). + +For more info about `hide_parameters`, see the next section on [Filter parameters](#filter-parameters). + +### Filter parameters + +You can display a filtered view of your question or dashboard in a public embed. Make sure you've set up a [question filter](../query-builder/introduction.md#filtering) or [dashboard filter](../../dashboards/filters.md) first. + +To apply a filter to your embedded question or dashboard, add a _query_ parameter to the end of the link in your iframe's `src` attribute, like this: + +``` +/dashboard/42?filter_name=value +``` + +For example, say that we have a dashboard with an "ID" filter. We can give this filter a value of 7: + +``` +/dashboard/42?id=7 ``` -You don't _have_ to assign a filter a value, though — if you only want to hide it, so that it isn't usable in this context, you can do this: +To set the "ID" filter to a value of 7 _and_ hide the "ID" filter widget from the public embed: ``` -/dashboard/42#hide_parameters=id +/dashboard/42?id=7#hide_parameters=id ``` -Note that the name of the filter in the URL should be specified in lower case, and with underscores instead of spaces. So if your filter was called "Filter for User ZIP Code," you'd write: +To specify multiple values for filters, separate the values with ampersands (&), like this: ``` -/dashboard/42#hide_parameters=filter_for_user_zip_code +/dashboard/42?id=7&name=janet ``` -You can specify multiple filters to hide by separating them with commas, like this: +You can hide multiple filter widgets by separating the filter names with commas, like this: ``` /dashboard/42#hide_parameters=id,customer_name ``` -To specify multiple values for filters, though, you'll need to separate them with ampersands (&), like this: +Note that the name of the filter in the URL should be specified in lower case, and with underscores instead of spaces. If your filter is called "Filter for User ZIP Code", you'd write: ``` -/dashboard/42?id=7&customer_name=janet +/dashboard/42?filter_for_user_zip_code=02116 ``` ## Further reading diff --git a/docs/questions/start.md b/docs/questions/start.md index cad23dd8e676..7ba622d428a3 100644 --- a/docs/questions/start.md +++ b/docs/questions/start.md @@ -32,10 +32,6 @@ Also known as the SQL editor (we say native because you can also query databases Write native code (like SQL) to query your data source. -### [Viewing metadata](./native-editor/data-model-reference.md) - -Metabase's data model reference can help you understand your tables and fields. - ### [SQL templates](./native-editor/sql-parameters.md) Pass parameters into variables in your SQL templates. @@ -60,6 +56,6 @@ Choose from a variety of visualization types. Get results via email or Slack, either on a schedule, or only when something interesting happens. -### [Public links](./sharing/public-links.md) +### [Public sharing](./sharing/public-links.md) -Share your data with the good people of the internet. +Create links or embeds for the good people of the internet. diff --git a/docs/troubleshooting-guide/cant-log-in.md b/docs/troubleshooting-guide/cant-log-in.md index 9baf22e5c49b..4590903ca6a5 100644 --- a/docs/troubleshooting-guide/cant-log-in.md +++ b/docs/troubleshooting-guide/cant-log-in.md @@ -4,93 +4,30 @@ title: People can't log in to Metabase # People can't log in to Metabase -## Do you know how your logins are managed? +## No access to Metabase login page -- [Metabase](#common-metabase-login-problems) -- [SAML][troubleshooting-saml] -- [LDAP][troubleshooting-ldap] -- [I don't know how my logins are managed](#i-dont-know-how-my-logins-are-managed). +If you're not a Metabase admin, you'll have to tag them for help here. -## Common Metabase login problems +1. Check that you have the correct [site URL](../configuring-metabase/settings.md) from **Settings** > **Admin settings** > **General**. +3. Check if the [account is deactivated](../people-and-groups/managing.md#deactivating-an-account). -- [I can't access the Metabase login page](#cant-access-the-metabase-login-page). -- [I can't log in to my Metabase Cloud account](#cant-log-in-to-a-metabase-cloud-account). -- [I forgot my password][how-to-reset-password]. -- [I forgot the admin password][how-to-reset-admin-password]. -- [I want to delete an account that was set up incorrectly][how-to-delete-an-account]. +## No access to Metabase Cloud account -### Can't access the Metabase login page +The admin password for `store.metabase.com` (where you can find payment and subscription info) is not necessarily the same as the password for your Metabase instance (where you log in to look at data). -1. Check whether you need to include a port number as well as a hostname in the connection URL. For example, Metabase might be at `https://example.com:3000/` instead of `https://example.com/`. - - If you're an administrator, you'll have configured this. - - If you're not, please ask your admin. -2. Check whether your Metabase instance has moved. For example, if you were using a trial instance of Metabase, but you're now in production, the URL might have changed. -3. [Ask your Metabase admin if your account has been deactivated][how-to-reactivate-account]. +If you've forgotten your Metabase Cloud admin password, you can [contact support](https://www.metabase.com/help-premium) to reset the password. -For more information, see the [Configuration settings documentation][config-settings]. +## Related topics -## Can't log in to a Metabase Cloud account - -If you're using [Metabase Cloud][pricing], note that your Metabase store password is different from your Metabase Cloud admin password. - -- [I forgot my Metabase store password][reset-store-password]. -- [I forgot my Metabase Cloud password][how-to-reset-admin-password]. -- If you're a Metabase Cloud customer, you can [contact support][help-premium]. - -For more information, see the [Metabase Cloud documentation][cloud-docs]. - -## I don't know how my logins are managed - -What do you use to log in? - -- **An email address and password.** - - You're using [Metabase](#common-metabase-login-problems) or [LDAP][troubleshooting-ldap]. - -- **A button that launches a [pop-up dialog][sso-gloss].** - - You're using SSO. If you're using SAML, go to [Troubleshooting SAML][troubleshooting-saml]. If you're using JWT, search or ask the [Metabase community][discourse]. - -- **I'm signing in at a site that doesn't have `.metabase.com` in the URL.** - - You're using an embedded application. If a Metabase question or dashboard is embedded in another website or web application, that site or application determines who you are. It may pass on your identity to Metabase to control what data you are allowed to view---please see [our troubleshooting guide for sandboxing][sandboxing] if you are having trouble with this. - -- **I'm signing in at `store.metabase.com`.** - - If you're using [Metabase Cloud][pricing], the password for the Metabase store (where you pay for things) is not automatically the same as the password for your Metabase instance (where you log in to look at data). - - If your password and URL are correct, go to [Can't log in to a Metabase Cloud account](#cant-log-in-to-a-metabase-cloud-account). - -## Further reading - -- [People and groups][people-and-groups] -- [SAML documentation][saml-docs] -- [LDAP documentation][ldap-docs] -- [Checking a person's auth method][how-to-find-auth-method-for-an-account] +- [Troubleshooting SAML](./saml.md). +- [Troubleshooting LDAP](./ldap.md). +- [Resetting someone's password](../people-and-groups/managing.md#resetting-someones-password). +- [Resetting the admin password](../people-and-groups/managing.md#resetting-the-admin-password). +- [Deleting an account that's set up incorrectly](../people-and-groups/managing.md#deleting-an-account). ## Are you still stuck? If you can’t solve your problem using the troubleshooting guides: -- Search or ask the [Metabase community][discourse]. -- Search for [known bugs or limitations][known-issues]. - -[cloud-docs]: https://www.metabase.com/cloud/docs/ -[config-settings]: ../configuring-metabase/settings.md -[discourse]: https://discourse.metabase.com/ -[help-premium]: https://www.metabase.com/help-premium -[how-to-delete-an-account]: ../people-and-groups/managing.md#deleting-an-account -[how-to-find-auth-method-for-an-account]: ../people-and-groups/managing.md#checking-someones-auth-method -[how-to-reactivate-account]: ../people-and-groups/managing.md#reactivating-an-account -[how-to-reset-admin-password]: ../people-and-groups/managing.md#resetting-the-admin-password -[how-to-reset-password]: ../people-and-groups/managing.md#resetting-someones-password -[known-issues]: ./known-issues.md -[ldap-docs]: ../people-and-groups/google-and-ldap.md#enabling-ldap-authentication -[people-and-groups]: ../people-and-groups/start.md -[pricing]: https://www.metabase.com/pricing -[reset-store-password]: https://store.metabase.com/forgot-password -[saml-docs]: ../people-and-groups/authenticating-with-saml.md -[sandboxing]: ./sandboxing.md -[sso-gloss]: https://www.metabase.com/glossary/sso -[troubleshooting-ldap]: ./ldap.md -[troubleshooting-saml]: ./saml.md +- Search or ask the [Metabase community](https://discourse.metabase.com/). +- Search for [known bugs or limitations](./known-issues.md). diff --git a/docs/troubleshooting-guide/cant-see-tables.md b/docs/troubleshooting-guide/cant-see-tables.md index 2c17860cc194..b4e773237426 100644 --- a/docs/troubleshooting-guide/cant-see-tables.md +++ b/docs/troubleshooting-guide/cant-see-tables.md @@ -31,15 +31,7 @@ Exactly what settings you need will depend on your environment. To test that the 1. Try to connect to the database using some other application (e.g., `psql` for PostgreSQL). -If you can't connect to the database with another application, the problem is probably not with Metabase. Please check that the database server is running and that you have the correct host, port, username, password, and other settings. - -## What's the status of your database connection? - -**Root cause:** You've configured a database connection, but it's not working as expected. - -**Steps to take:** - -If your connection isn't working, go to [Troubleshooting database connections](./db-connection.md). +If you can't connect to the database with another application, the problem is probably not with Metabase. Please check that the database server is running and that you have the correct host, port, username, password, and other settings. For more help, see [Troubleshooting database connections](./db-connection.md). ## Does the table exist? @@ -59,7 +51,7 @@ Table "SOMEWHERE" not found If you see this message, use another application (e.g., `psql` for PostreSQL) to send the same query to the database. If it also produces a "table not found" message, check the database schema and the spelling of the table name. -Be sure to log in to the database using the same credentials that Metabase uses. A common problem is that the account Metabase uses to connect to the database lacks the same privileges as a member of IT staff or a developer, so tables that are visible to the latter when they use external applications are invisible to Metabase. You may want to [check if Metabase has the correct permissions](./data-permissions.md#getting-a-permission-denied-error-message). +Be sure to log in to the database using the same credentials that Metabase uses. A common problem is that the account Metabase uses to connect to the database lacks the same privileges as a member of IT staff or a developer, so tables that are visible to the latter when they use external applications are invisible to Metabase. For more help, see [Troubleshooting syncs, scans, and fingerprinting](./sync-fingerprint-scan.md). ## Does the person who cannot see the table have permission to view it? @@ -71,23 +63,14 @@ Be sure to log in to the database using the same credentials that Metabase uses. 2. Confirm that the tables are not visible. 3. Log out, then log in using the administrator's credentials. -If the administrator's account can see the tables but an individual person cannot, see [Troubleshooting permissions][troubleshooting-permissions]. - -## Is Metabase's metadata out of sync with the state of the database? +If the administrator's account can see the tables but an individual person cannot, see [Troubleshooting data permissions](./data-permissions.md). -**Root cause:** In order to display available tables and columns in dropdown menus and previews, Metabase runs a query every hour to find out what tables are available and what columns are in each available table, and stores this information in its application database. +## MongoDB -1. If a table has been added or removed since the last time this "sync" operation ran, Metabase's information about the database will be outdated. -2. In some rare cases Metabase may time out while synchronizing with the database. For example, if you're using MongoDB and have very large (hundreds of kilobytes) JSON blobs, the sync operation may not complete in the allowed time. - -**Steps to take:** +MongoDB lets you "successfully connect" to any collection name, even the collection doesn't exist. If you don't see a MongoDB collection in Metabase, make sure that: -1. Run the "sync" process manually: - 1. Go to Admin Panel > Databases. - 2. Choose the database. - 3. Click on "Sync database schema now". -2. Go to Admin > Troubleshooting > Logs and see if there are any error messages saying that the "sync" operation could not run (e.g., because the network or the database itself was temporarily down). -3. If there are no suspicious error messages, log out of Metabase, close the browser tab, log back into Metabase in a new browser tab, and try to access your table again. +- you have the correct collection name, and +- the collection is non-empty. ## Related problems diff --git a/docs/troubleshooting-guide/cant-view-or-edit.md b/docs/troubleshooting-guide/cant-view-or-edit.md index fc78cedeefa6..0f4c7af088d0 100644 --- a/docs/troubleshooting-guide/cant-view-or-edit.md +++ b/docs/troubleshooting-guide/cant-view-or-edit.md @@ -29,11 +29,11 @@ Someone with edit permissions (your Metabase admin is probably a safe bet) will ## Related problems +- [Error message: your question took too long](./timeout.md). +- [Error message: permission denied](./data-permissions.md#getting-a-permission-denied-error-message). - [I can't view or edit queries from the SQL editor](./data-permissions.md#a-user-group-cant-access-the-sql-editor). -- [I'm getting a "permission denied" error message](./data-permissions.md#getting-a-permission-denied-error-message). - [I can't save my question or dashboard](./proxies.md). - [I can't see my tables](./cant-see-tables.md). -- [My visualizations are wrong](./visualization.md). ## Are you still stuck? diff --git a/docs/troubleshooting-guide/data-permissions.md b/docs/troubleshooting-guide/data-permissions.md index 6a5e941a2b2a..ac7099766726 100644 --- a/docs/troubleshooting-guide/data-permissions.md +++ b/docs/troubleshooting-guide/data-permissions.md @@ -79,7 +79,7 @@ If you get an error message that says something like "permission denied to \; +GRANT ALL ON IN SCHEMA TO ; +``` + +To allow Metabase to query all tables in a specific schema: + +```sql +USE ; +GRANT ALL ON
TO ; +``` + ## Do you have a different problem? - [I can't view or edit my question or dashboard][view-edit]. diff --git a/docs/troubleshooting-guide/filters.md b/docs/troubleshooting-guide/filters.md index 9fb3ba3b93a9..0d8ad31df3f0 100644 --- a/docs/troubleshooting-guide/filters.md +++ b/docs/troubleshooting-guide/filters.md @@ -1,72 +1,88 @@ --- -title: My dashboard filters don't work +title: Troubleshooting filters --- -# My dashboard filters don't work -You've tried to add a [filter widget][filter-widget-gloss] to your dashboard, but: +# Troubleshooting filters -- the question you want to connect the filter to doesn't show up, or -- the filter doesn't show a dropdown list of possible values when you use it, or -- the filter removes all of the rows from the table no matter what value you set it to. +It's always a good idea to start with a quick sanity check: -If you've created a [linked filter][linked-filter-gloss], please see [this troubleshooting guide][troubleshoot-linked-filters] instead. +1. Clear your browser cache. +2. Refresh the page. +3. Open your question or dashboard in an incognito window. -If you're using a [SQL variable][sql-variable-gloss] with the variable type "field filter", go to [Troubleshooting SQL variables][troubleshoot-sql-variables]. +## Dashboard filters -## Is the dashboard filter actually connected to your question? +1. Click the **pencil** icon to go into edit mode. +2. Click the **gear** icon beside your filter widget. +3. Make sure you've selected a column for your filter under **Column to filter on**. +4. If you can't find the right **Column to filter on**, or you're getting "No Results" when you apply the filter: + - Exit edit mode and click on a dashboard card to go to the _original question_. + - Follow the troubleshooting steps under [Question filters](#question-filters). -**Root cause:** The filter isn't connected to any cards on the dashboard, or connected to the wrong field. +## Question filters -**Steps to take:** +1. Make sure the question includes the column you want to filter on. +2. Check that the column actually contains the value(s) you're filtering on. You can do this by: + - sorting number or date columns, + - creating a "contains" filter for string columns, or + - asking your database admin. +3. Ask your Metabase admin to help you check if: + - Metabase is [up to date](../databases/connecting.md#manually-syncing-tables-and-columns) with your database, + - the column is [visible](../data-modeling/metadata-editing.md#column-visibility) in Metabase, + - you have the correct [data permissions](../permissions/data.md) to access the column. -1. In dashboard edit mode, click on the gear icon next to the filter. Check that each card you want to wire up to the filter has a column selected. -2. If no columns are available to select on that card, you may need to change the filter type, from say a text filter to a date filter, to connect the filter to the card. -3. Check that the filter widget is connected to the column you want to filter on each relevant card. +### Special cases -## If the card you're trying to filter is written in SQL, does its SQL query contain a variable? +If you're having trouble filtering on a: -**Root cause**: If your SQL question doesn't contain a variable, the filter can't insert the value into the query to filter the results. +- [Custom column](../questions/query-builder/introduction.md#creating-custom-columns): check if the custom expression is working as expected. For example, your custom expression might be returning blank values when you expect numbers. +- [SQL field filter](../questions/native-editor/sql-parameters.md#the-field-filter-variable-type): make sure you're using the correct [field filter syntax](../questions/native-editor/sql-parameters.md#field-filter-syntax), then see [Troubleshooting SQL variables](./sql.md#field-filter-variables). -**Steps to take**: +**Explanation** -1. Check that your SQL query contains at least [one variable][sql-variable-gloss] for the filter to insert the value. These can be plain variables, or [Field Filters][field-filter], with names enclosed in double curly braces `{% raw %}{{variable_name}}{% endraw %}`, typically in a `WHERE` clause. -2. If these steps don’t fix your error, go to [Troubleshooting SQL variables][troubleshoot-sql-variables]. +When we first set up a filter, we need to link the filter to a column. If we make the wrong assumptions about a column's values or data type, the filter won't work at all. If a column changes, the filter might suddenly stop working. -If you built your question in the Query Builder, Metabase knows which columns you're using, and which columns you can connect to different types of filters. So you can add a dashboard filter and refer to columns in the question's results without creating variables explicitly. +For example, let's say we want to create a filter named "Select Product ID" linked to a column named **Product ID**. The filter won't work if any of these things happen: -## Are you seeing a different kind of input widget than you expected? +- Our question doesn't include the **Product ID** column. +- We type the number 4 into the "Select Product ID" filter, when the **Product ID** column only contains the values 1, 2, and 3. +- **Product ID** is renamed to something else in the database or Data Model page. +- **Product ID** is deleted from the database, or hidden in the Data Model page. +- **Product ID** is a custom column that's not working as expected. +- We don't have data permissions to access the **Product ID** column. +- We made "Select Product ID" a numerical filter, but **Product ID** is a string column (see the section below). -For example, you want a dropdown but you're seeing a search box or a text input box. +## Time, ID, and number filters -**Root cause:** Metabase only displays a dropdown list of possible values for a variable if it knows that the field in question is a category rather than (for example) an arbitrary number or arbitrary text. However, if the number of unique categories exceeds 100 values, Metabase will display a search box with autocomplete instead of a dropdown. +If you're not a Metabase admin, you might have to ask your admin to help you with this. -**Steps to take:** +1. Find the [data type](https://www.metabase.com/learn/databases/data-types-overview) of the column that you want to filter on. You can find this info from: + - the [Data reference](../exploration-and-organization/data-model-reference.md), + - the [Data Model page](../data-modeling/metadata-editing.md) (admins only), or + - directly from the database. +2. Cast the column to a data type that matches the desired [filter type](../questions/query-builder/introduction.md#filter-types). You can: + - [cast strings or numbers to dates](../data-modeling/metadata-editing.md#casting-to-a-specific-data-type) from the Data Model page, or + - change the data type of the column in your database, and [re-sync](../databases/connecting.md#manually-syncing-tables-and-columns) the database schema. -1. Go to the **Admin Panel** and select the **Data Model** tab. -2. Select the database, schema, table, and field in question. -3. Click the gear-icon to view all the field's settings. -4. Set **Field Type** to "Category" and **Filtering on this field** to "A list of all values." -5. Click the button **Re-scan this field** in the bottom. +**Explanation** -If you created the question in SQL, then you only get a dropdown if the filter is a Field Filter _and_ the Filtering on this field option is set to your preferred input type: A list of all values (dropdown list) _and_ the number of unique values is less than 100. +Metabase needs to know the data type of a column in order to present you with a curated selection of filter types. Sometimes these columns are mistyped---if a column stores your numbers as strings, Metabase will only show you text or category filters (with options like "is", "is not") instead of number filters (with options like "greater than", "less than"). -## Has someone renamed or deleted columns in the database? +Timestamps, in particular, are the root of all evil, so please be patient with your Metabase admin (or yourself!) when trying to get the data type right. -**Root cause:** Someone has changed the database schema, e.g., renamed or deleted a column in a table. +## Related topics -**Steps to take:** +- [Troubleshooting linked filters](./linked-filters.md) +- [Troubleshooting SQL variables](./sql.md#field-filter-variables) +- [Troubleshooting dates and times](./timezones.md) +- [Creating dropdown filters](../data-modeling/metadata-editing.md#changing-a-search-box-filter-to-a-dropdown-filter) +- [Creating SQL filters](../questions/native-editor/sql-parameters.md#the-field-filter-variable-type) +- [Field filter gotchas](../questions/native-editor/sql-parameters.md#field-filter-gotchas) -If a filter that used to work no longer seems to, or seems to eliminate all of the rows: +## Are you still stuck? -1. [Re-sync][sync-scan] Metabase with the database (i.e., refresh Metabase's understanding of the database's structure). -2. Compare the names of the fields used in the question with the actual names of the fields in the database. -3. Modify the question to match the current database schema. +If you can’t solve your problem using the troubleshooting guides: -[field-filter]: https://www.metabase.com/learn/sql-questions/field-filters -[filter-widget-gloss]: https://www.metabase.com/glossary/filter_widget -[linked-filter-gloss]: https://www.metabase.com/glossary/linked_filter -[sql-variable-gloss]: https://www.metabase.com/glossary/variable#example-variable-in-metabase -[sync-scan]: ./sync-fingerprint-scan.md -[troubleshoot-linked-filters]: ./linked-filters.md -[troubleshoot-sql-variables]: ./sql.md#my-sql-variables-arent-working +- Search or ask the [Metabase community](https://discourse.metabase.com/). +- Search for [known bugs or limitations](./known-issues.md). diff --git a/docs/troubleshooting-guide/index.md b/docs/troubleshooting-guide/index.md index 545124a53952..19514ba508d4 100644 --- a/docs/troubleshooting-guide/index.md +++ b/docs/troubleshooting-guide/index.md @@ -39,8 +39,8 @@ Problems, their causes, how to detect them, and how to fix them. - [My dashboard is slow or failing to load][slow-dashboard]. - [My SQL question doesn't work][sql]. - [The dates and times in my questions and charts are wrong][incorrect-times]. -- [My dashboard filters don't work][filters]. -- [My dashboard's linked filters don't work][linked-filters]. +- [My filters don't work][filters]. +- [My linked filters don't work][linked-filters]. ## Models diff --git a/docs/troubleshooting-guide/models.md b/docs/troubleshooting-guide/models.md index ced004f9d090..51f2d0ee1fdb 100644 --- a/docs/troubleshooting-guide/models.md +++ b/docs/troubleshooting-guide/models.md @@ -66,7 +66,7 @@ If you can’t solve your problem using the troubleshooting guides: [materialize-views-learn]: https://www.metabase.com/learn/administration/making-dashboards-faster.html#materialize-views-create-new-tables-to-store-query-results [model-button-image]: https://www.metabase.com/learn/images/models/model-icon.png [model-docs]: ../data-modeling/models.md -[nested-query-settings-docs]: ../configuring-metabase/settings.md#enabled-nested-queries +[nested-query-settings-docs]: ../configuring-metabase/settings.md#enable-nested-queries [replicate-database-learn]: https://www.metabase.com/learn/administration/making-dashboards-faster.html#replicate-your-database [sql-explain-learn]: https://www.metabase.com/learn/sql-questions/sql-best-practices.html#explain [summary-tables-learn]: https://www.metabase.com/learn/administration/making-dashboards-faster.html#aggregate-data-ahead-of-time-with-summary-tables diff --git a/docs/troubleshooting-guide/my-dashboard-is-slow.md b/docs/troubleshooting-guide/my-dashboard-is-slow.md index 156e9b464e00..2a0e718b966c 100644 --- a/docs/troubleshooting-guide/my-dashboard-is-slow.md +++ b/docs/troubleshooting-guide/my-dashboard-is-slow.md @@ -42,7 +42,7 @@ Caching takes less effort because it doesn't involve any changes to your schemas **Explanation** -One of the easiest ways to make a question or dashboard run faster is to work with a smaller dataset. Your Metabase admin can apply automatic data limitations using things like [SSO](../people-and-groups/start.md#setting-up-single-sign-on-sso), [data permissions](../permissions/data.md), and [data sandboxing](../permissions/data-sandboxes.md). +One of the easiest ways to make a question or dashboard run faster is to work with a smaller dataset. Your Metabase admin can apply automatic data limitations using things like [SSO](../people-and-groups/start.md#authentication), [data permissions](../permissions/data.md), and [data sandboxing](../permissions/data-sandboxes.md). When someone loads a question or a dashboard in a signed embed, however, that question or dashboard will query the full dataset (rather than a smaller dataset limited by permissions). Standalone, [signed embeds](../embedding/signed-embedding.md) don't require people to be logged in, and unauthenticated people viewing the signed embed won't be subject to the permissions and data restrictions set up by your admin. diff --git a/docs/troubleshooting-guide/running.md b/docs/troubleshooting-guide/running.md index 95fc9adc600b..9db00c6e421d 100644 --- a/docs/troubleshooting-guide/running.md +++ b/docs/troubleshooting-guide/running.md @@ -6,6 +6,10 @@ title: Running Metabase Metabase runs on the Java Virtual Machine (JVM), and depending on how it's configured, it may use the server's filesystem to store some information. Problems with either the JVM or the filesystem can therefore prevent Metabase from running. +## Java version + +Make sure you're using a Java version of 11 or higher. For more info, see [Java versions](../installation-and-operation/java-versions.md#check-installed-version). + ## WARNING: sun.reflect.Reflection.getCallerClass is not supported Don't worry about it. diff --git a/docs/troubleshooting-guide/sql.md b/docs/troubleshooting-guide/sql.md index d121626f0787..5bfd0bebc865 100644 --- a/docs/troubleshooting-guide/sql.md +++ b/docs/troubleshooting-guide/sql.md @@ -26,10 +26,10 @@ What type of [SQL variable][sql-variable-def] are you using? ### Field filter variables -- [Filter widget doesn't display a dropdown menu of values](./filters.md#are-you-seeing-a-different-kind-of-input-widget-than-you-expected). +- [Filter widget doesn't display a dropdown menu of values](../data-modeling/metadata-editing.md#changing-a-search-box-filter-to-a-dropdown-filter). - [SQL query contains a subquery (nested query) or CTE](../questions/native-editor/sql-parameters.md#field-filters-dont-work-with-table-aliases). - [400 error from BigQuery](../questions/native-editor/sql-parameters.md#some-databases-require-the-schema-in-the-from-clause). -- [SQL syntax error: missing `FROM` clause](../questions/native-editor/writing-sql.md#how-metabase-executes-sql-variables). +- [SQL syntax error: missing `FROM` clause](../questions/native-editor/sql-parameters.md#include-dependencies-in-your-query). ### Text, number, or date variables diff --git a/docs/troubleshooting-guide/sync-fingerprint-scan.md b/docs/troubleshooting-guide/sync-fingerprint-scan.md index 4c019077dddc..56a1cf104430 100644 --- a/docs/troubleshooting-guide/sync-fingerprint-scan.md +++ b/docs/troubleshooting-guide/sync-fingerprint-scan.md @@ -1,85 +1,158 @@ --- -title: Troubleshooting syncs and scans +title: Troubleshooting syncs, scans, and fingerprinting --- -# Troubleshooting syncs and scans +# Troubleshooting syncs, scans, and fingerprinting -Metabase needs to know what's in your database in order to show tables and fields, populate dropdown menus, and suggest good visualizations, but loading all the data would be very slow (or simply impossible if you have a lot of data). It therefore does three things: +First, check if your data is outdated because of browser caching: -1. Metabase periodically asks the database what tables are available, then asks which columns are available for each table. We call this *syncing*, and it happens [hourly or daily][sync-frequency] depending on how you've configured it. It's very fast with most relational databases, but can be slower with MongoDB and some [community-built database drivers][community-db-drivers]. +1. Clear your browser cache. +2. Refresh your Metabase page. +3. Open your Metabase page in an incognito window. -2. Metabase *fingerprints* the column the first time it synchronizes. Fingerprinting fetches the first 10,000 rows from each column and uses that data to guesstimate how many unique values each column has, what the minimum and maximum values are for numeric and timestamp columns, and so on. Metabase only fingerprints each column once, unless the administrator explicitly tells it to fingerprint the column again, or in the rare event that a new release of Metabase changes the fingerprinting logic. +Once you've confirmed that you're looking at a non-cached view of your tables and columns, tag your database admin for help with troubleshooting: -3. A *scan* is similar to fingerprinting. Metabase will scan a database by default every 24 hours (though you can configure Metabase to run a scan less frequently, or disable scanning entirely). When you set a field to "A list of all values" in the [Data Model](../data-modeling/metadata-editing.md), which is used to display options in dropdown menus, scanning looks at the first 1,000 distinct records (ordered ascending). For each field scanned, Metabase stores only the first 100 kilobytes of text. If more values exist, Metabase displays the stored values in the dropdown menus, and only triggers a database search query to look for more values when people type in the search box for that filter widget. +- **Syncs**, if your tables or columns are missing, or your column data types are wrong. +- **Scans**, if your column _values_ are missing or wrong (for example, in your filter dropdown menus). +- **Fingerprinting**, if you've triggered a manual scan, but the changes aren't taking effect. -## Metabase can't sync, fingerprint, or scan +## Syncing -If the credentials Metabase is using to connect to the database don't give it privileges to read the tables, the first sign will often be a failure to sync, which would then also stop fingerprint and scan. +1. Make sure your database driver is up to date. +2. Go to **Admin** > **Troubleshooting** > **Logs** to check the status of the sync. +3. Run a query against your database from the Metabase SQL editor to check for database connection or database privilege errors that aren't listed in the logs: -**How to detect this:** You can't see any of the tables in the database, or columns that have just been added to your data source don't show up in Metabase. + ```sql + SELECT * + FROM "your_schema"."your_table_or_view" + LIMIT 1 + ``` +5. [Manually re-sync](../databases/connecting.md#manually-syncing-tables-and-columns) the table or view if needed. -**How to fix this:** [This guide][troubleshooting-db-connection] explains how to troubleshoot database connections. The relevant steps for solving this problem are: +### Special cases -1. Sometimes browsers will show an old cached list of tables or columns. Refreshing the page will update the cache. -2. If you've just set up a new database in Metabase, the sync process might still be running---it's normally fast, but it can sometimes take a while. You can follow its progress in Admin > Troubleshooting > Logs. -3. If you've just added a table or a column, Metabase might not have synced yet. You can manually run the sync process by going to the Admin Panel, selecting "Databases", choosing your database, and clicking on "Sync database schema now". -4. To see if the problem is caused by lack of database privileges, try running a query like the one below for each table you think you should be able to access : +If you’ve just set up a new database in Metabase, the initial sync query needs some time to kick off. If the sync hasn't started at all, try [Troubleshooting database connections](./db-connection.md). +**Explanation** + +A sync query should show up like this in your database's query execution table (using the privileges for the database user in the database connection details): + +```sql +SELECT TRUE +FROM "your_schema"."your_table_or_view" +WHERE 1 <> 1 +LIMIT 0 ``` -SELECT * -FROM table -LIMIT 1 + +To run the sync query, Metabase must: + +- successfully connect to your database, and +- be [granted privileges](./data-permissions.md#granting-database-privileges) to query that database. + +If the [connection is failing](./db-connection.md) or the database privileges are wrong, the sync query won't be able to run. If Metabase can't sync with your database after you first set it up, then the initial scan and fingerprinting queries won't run either. + +## Unfolding JSON columns with Object records + +1. Go to **Admin** > **Databases** > **your database** > **Show advanced options**. +2. Click **Disable "JSON unfolding"** +3. Click **Save changes**. +4. Click **Sync database schema**. + +**Explanation** + +Metabase will try to unfold JSON and JSONB records during the sync process, which can take up a decent chunk of query execution time. If you have a lot of JSON records, try disabling the automatic unfolding option to pull the sync out of slow-motion. Remember that you can follow the status of the sync from **Admin** > **Troubleshooting** > **Logs**. + +## Scanning + +1. Go to **Admin** > **Data Model**. +2. Select the database and table. +3. Go to the column you want to update, and click the **gear** icon. +4. Click **Discard cached field values**. +5. Click **Re-scan this field**. +6. Go to **Admin** > **Troubleshooting** > **Logs** to follow the status of the scan and debug errors from there. + +### Special cases + +If you're waiting for the initial scan to run after connecting a database, make sure the initial sync has completed first (remember you can check the status from **Admin** > **Troubleshooting** > **Logs**). + +**Explanation** + +Scan queries are run against your database to sample column values from the first 1,000 rows in a table or view: + +```sql +SELECT "your_table_or_view"."column" AS "column" +FROM "your_schema"."your_table_or_view" +GROUP BY "your_table_or_view"."column" +ORDER BY "your_table_or_view"."column" ASC +LIMIT 1000 ``` -Note that we only get the first 10,000 documents when scanning a MongoDB collection, so if you're not seeing some new fields, those fields might not exist in the documents we looked at. Please see [this discussion][metabase-mongo-missing] for more details. +A failed scan is caused by a failed scan query---you can look at the logs to debug the query similar to other queries you'd run directly against your database. + +Note that when you [change a search box filter to a dropdown filter](../data-modeling/metadata-editing.md#changing-a-search-box-filter-to-a-dropdown-filter) from the Data Model, you'll trigger a scan query for that field. If you have a dropdown filter that isn't picking up all the values in a field, remember that Metabase only samples the first 1,000 unique values per field, and stores a maximum of 100 kilobytes of text. If you've got more than 1,000 unique values in a column, or a lot of text-heavy data (like long URLs or survey responses), you can: + +- Use a search box filter for that field. +- Clean up the data further in your [ETL or ELT](https://www.metabase.com/learn/analytics/etl-landscape) process. + +## Fingerprinting -## Metabase isn't showing all of the values I expect to see +To manually re-trigger a fingerprinting query for a given column: -**How to detect this:** +1. Go to **Admin** > **Databases** > **your database** > **Show advanced options**. +2. Toggle ON **Periodically refingerprint tables** and click **Save changes**. +3. Go to **Admin** > **Data Model**. +4. Select your database and table. +5. Change the visibility of the table to "Hidden". +6. Change the visibility back to "Queryable". +7. Wait 10 seconds. +8. Go to your column and change the **Type** from "Entity Key" to "No semantic type", and back to "Entity Key". -1. The UI isn't displaying some of the values you expect to see in a dropdown menu. -2. The UI is showing a search box for selecting values where you expect a dropdown menu. +### Special cases -**How to fix this:** +If you're waiting for the initial fingerprinting query to run after connecting a database, make sure the initial sync has completed first (remember you can check the status from **Admin** > **Troubleshooting** > **Logs**). + +If you're using MongoDB, Metabase fingerprints the first 10,000 documents per collection. If you're not seeing all of your fields, it's because those fields might not exist in those first 10,000 documents. For more info, see our [MongoDB reference doc](../databases/connections/mongodb.md#i-added-fields-to-my-database-but-dont-see-them-in-metabase). + +**Explanation** + +The initial fingerprinting query looks at the first 10,000 rows from a given table or view in your database: + +```sql +SELECT * +FROM "your_schema"."your_table_or_view" +LIMIT 10000 +``` -1. Go to the Admin Panel and select the **Data Model** tab. -2. Select the database, schema, table, and field in question. -3. Click the gear-icon to view all the field's settings. -4. Set **Field Type** to "Category" and **Filtering on this field** to "A list of all values." -5. Click the button **Re-scan this field** in the bottom. +If the first 10,000 rows aren't representative of the data in a table (for example, if you've got sparse data with a lot of blanks or nulls), you could see issues such as: -## I cannot force Metabase to sync or scan using the API +- Incorrect [filter types](../questions/query-builder/introduction.md#filter-types), such as a category when you want a calendar. +- Histogram visualizations that don't work (since Metabase needs a min and max value to generate the bins). -Metabase syncs and scans regularly, but if the database administrator has just changed the database schema, or if a lot of data is added automatically at specific times, you may want to write a script that uses the [Metabase API][api-learn] to force sync or scan to take place right away. [Our API][metabase-api] provides two ways to do this: +Metabase doesn't have a built-in option to trigger manual fingerprinting queries. You can "reset" a field's settings using the steps above to try and force a fingerprinting query, but it's not guaranteed to work on all versions of Metabase. -1. Using an endpoint with a session token: `/api/database/:id/sync_schema` or `api/database/:id/rescan_values`. These do the same things as going to the database in the Admin Panel and choosing **Sync database schema now** or **Re-scan field values now** respectively. In this case you have to authenticate with a user ID and pass a session token in the header of your request. +## Syncing or scanning is taking a long time -2. Using an endpoint with an API key: `/api/notify/db/:id`. This endpoint was made to notify Metabase to sync after an [ETL operation][etl] finishes. In this case you must pass an API key by defining the `MB_API_KEY` environment variable. +To speed up **syncs**: + - Restrict the privileges used to connect to the database so that Metabase only syncs a limited subset of schemas or tables. + - [Reduce the frequency of sync queries](../databases/connecting.md#scheduling-database-scans). -**How to detect this:** Your script fails to run. +To speed up **scans**: + - [Reduce the frequency of scans, or disable scans entirely](../databases/connecting.md#scheduling-database-scans). + - Reduce the number of columns being scanned by going to **Admin** > **Data Model** and setting **Filtering on this field** to **Search box** or **Plain input box**. -**How to fix this:** +**Explanation** -1. Make sure you are able to sync and scan manually via the Admin Panel. -2. Make sure you're using the correct URL to send the request to Metabase. -3. Check the error message returned from Metabase. -4. Check the credentials you're using to authenticate and make sure they identify your script as a user with administrative privileges. +Syncs and scans are ultimately just two kinds of queries that are run against your database, so the speed of execution is limited by the number of queries that are run, the frequency of execution, the size of your data, and the amount of resources you've allocated to your database. Metabase gives you options to adjust the number and frequency of sync and scan queries, since unfortunately, we can't imbue your database with more power... (yet?) -## Sync and scan take a very long time to run +## Related topics -**How to detect this:** Sync and scan take a long time to complete. +- [Troubleshooting database connections](./db-connection.md). +- [Troubleshooting filters](./filters.md). +- [How syncs and scans work](../databases/connecting.md#syncing-and-scanning-databases). -**How to fix this:** -1. For sync, delays are usually caused by a large database with hundreds of schema, thousands of table and with hundreds of columns in each table. If you only need a subset of those tables or columns in Metabase, then restricting the privileges used to connect to the database will make sure that Metabase can only sync a limited subset of the database. -2. Scanning normally takes longer than sync, but you can reduce the number of fields Metabase will scan by changing the number of fields that have the **Filtering on this field** option set to "A list of all values". Setting fields to either "Search box" or "Plain input box" will exclude those fields from scans. +## Are you still stuck? -You can "fix" this by disabling scan entirely by going to the database in the Admin Panel and telling Metabase, "This is a large database," and then going to the Scheduling tab. However, sync is necessary: without it, Metabase won't know what tables exist or what columns they contain. +If you can’t solve your problem using the troubleshooting guides: -[api-learn]: https://www.metabase.com/learn/administration/metabase-api -[bugs]: ./bugs.md -[community-db-drivers]: ../developers-guide/partner-and-community-drivers.md -[etl]: https://www.metabase.com/glossary/etl -[metabase-api]: ../api-documentation.md -[metabase-mongo-missing]: ../databases/connections/mongodb.md#i-added-fields-to-my-database-but-dont-see-them-in-metabase -[sync-frequency]: ../databases/connecting.md#scheduling-database-syncs -[troubleshooting-db-connection]: ./db-connection.md +- Search or ask the [Metabase community](https://discourse.metabase.com/). +- Search for [known bugs or limitations](./known-issues.md). diff --git a/docs/troubleshooting-guide/timezones.md b/docs/troubleshooting-guide/timezones.md index e58b2395ff8d..3b02620c3039 100644 --- a/docs/troubleshooting-guide/timezones.md +++ b/docs/troubleshooting-guide/timezones.md @@ -44,14 +44,8 @@ Once you think you have identified a problem, drill down to understand exactly w **Steps to take:** -1. Go to the Admin Panel, select the **Localization** tab, and check the **Report Time Zone** setting, which controls the timezone Metabase uses when connecting to the database. This setting is currently supported on: - - Druid - - MySQL - - Oracle - - PostgreSQL - - Presto - - Vertica -2. If you're using a database that doesn't support a Report Time Zone, ensure that Metabase's time zone matches that of the database. Metabase's time zone is the Java Virtual Machine's time zone, typically set via a `-Duser.timezone<..>` parameter or the `JAVA_TIMEZONE` environment variable; exactly how it is set will depend on how you launch Metabase. Note that Metabase's time zone doesn't impact any databases that use a Report Time Zone. +1. Check the [report timezone setting](../configuring-metabase/localization.md#report-timezone) from **Admin settings** > **Settings** > **Localization**. +2. If you're using a database that doesn't support the report timezone setting, ensure that Metabase's time zone matches that of the database. Metabase's time zone is the Java Virtual Machine's time zone, typically set via a `-Duser.timezone<..>` parameter or the `JAVA_TIMEZONE` environment variable; exactly how it is set will depend on how you launch Metabase. Note that Metabase's time zone doesn't impact any databases that use a Report Time Zone. ## Are SQL queries not respecting the Reporting Time Zone setting? @@ -59,7 +53,15 @@ Once you think you have identified a problem, drill down to understand exactly w **Steps to take:** -1. Set a reporting time zone explicitly in your SQL query. +Set a reporting time zone explicitly in your SQL query. + +For example, you can write something like this with PostgreSQL: + +```sql +SELECT column::TIMESTAMP AT TIME ZONE 'EST' AS column_est +``` + +This statement casts the column to a `timestamp` data type first, then converts the `timestamp` into a `timestamptz` data type, with time zone 'EST'. ## Are dates without an explicit time zone being converted to another day? @@ -67,7 +69,7 @@ Once you think you have identified a problem, drill down to understand exactly w **Steps to take:** -1. Look at every time field your question uses in the [Data Model Reference][data-model] and see if any of them are simply a "Date" field. +1. Look at every time field your question uses in the [Data Model Reference](../exploration-and-organization/data-model-reference.md) and see if any of them are simply a "Date" field. 2. If so, make sure the server time zone reflects the reporting time zone, because when a query is run on Metabase, the server applies the configured time zone to that date. ## Are you mixing explicit and implicit time zones? @@ -78,5 +80,3 @@ Once you think you have identified a problem, drill down to understand exactly w 1. This typically happens with a question that uses multiple fields: for example, you're filtering on one timestamp and grouping by another. Check the time zones of each of the dates or times you are using in your question. 2. You'll need to explicitly set the time zone for any value that lacks an explicit time zone. This will need to be done either in a SQL query or by transforming the data in your database to ensure both timestamps have time zones. - -[data-model]: ../questions/native-editor/data-model-reference.md diff --git a/enterprise/backend/src/metabase_enterprise/config_from_file/core.clj b/enterprise/backend/src/metabase_enterprise/advanced_config/file.clj similarity index 83% rename from enterprise/backend/src/metabase_enterprise/config_from_file/core.clj rename to enterprise/backend/src/metabase_enterprise/advanced_config/file.clj index 9b7e5541273e..b54972239ade 100644 --- a/enterprise/backend/src/metabase_enterprise/config_from_file/core.clj +++ b/enterprise/backend/src/metabase_enterprise/advanced_config/file.clj @@ -1,5 +1,4 @@ -(ns ^{:added "0.45.0"} - metabase-enterprise.config-from-file.core +(ns ^{:added "0.45.0"} metabase-enterprise.advanced-config.file "Support for initializing Metabase with configuration from a `config.yml` file located in the current working directory. See https://github.com/metabase/metabase/issues/2052 for more information. @@ -99,26 +98,27 @@ [clojure.tools.logging :as log] [clojure.walk :as walk] [environ.core :as env] - [metabase-enterprise.config-from-file.databases] - [metabase-enterprise.config-from-file.interface :as config-from-file.i] - [metabase-enterprise.config-from-file.settings] - [metabase-enterprise.config-from-file.users] + [metabase-enterprise.advanced-config.file.databases] + [metabase-enterprise.advanced-config.file.interface :as advanced-config.file.i] + [metabase-enterprise.advanced-config.file.settings] + [metabase-enterprise.advanced-config.file.users] [metabase.driver.common.parameters] [metabase.driver.common.parameters.parse :as params.parse] + [metabase.public-settings.premium-features :as premium-features] [metabase.util :as u] [metabase.util.files :as u.files] - [metabase.util.i18n :refer [trs]] + [metabase.util.i18n :refer [trs tru]] [yaml.core :as yaml])) (comment ;; for parameter parsing metabase.driver.common.parameters/keep-me ;; for `settings:` section code - metabase-enterprise.config-from-file.settings/keep-me + metabase-enterprise.advanced-config.file.settings/keep-me ;; for `databases:` section code - metabase-enterprise.config-from-file.databases/keep-me + metabase-enterprise.advanced-config.file.databases/keep-me ;; for `users:` section code - metabase-enterprise.config-from-file.users/keep-me) + metabase-enterprise.advanced-config.file.users/keep-me) (set! *warn-on-reflection* true) @@ -127,7 +127,7 @@ map? (fn validate-section-configs [m] (doseq [[section-name section-config] m - :let [spec (config-from-file.i/section-spec section-name)]] + :let [spec (advanced-config.file.i/section-spec section-name)]] (s/assert* spec section-config)) true))) @@ -237,14 +237,30 @@ (s/assert* ::config m) (expand-templates m))) +(defn- sort-by-initialization-order + "Sort the various config sections. The `:settings` section should always be applied first (important, since it can + affect the other sections)." + [config-sections] + (let [{settings-sections true, other-sections false} (group-by (fn [[section-name]] + (= section-name :settings)) + config-sections)] + (concat settings-sections other-sections))) + (defn ^{:added "0.45.0"} initialize! "Initialize Metabase according to the directives in the config file, if it exists." [] ;; TODO -- this should only do anything if we have an appropriate token (we should get a token for testing this before ;; enabling that check tho) (when-let [m (config)] - (doseq [[section-name section-config] (:config m)] + (doseq [[section-name section-config] (sort-by-initialization-order (:config m))] + ;; you can only use the config-from-file stuff with an EE/Pro token with the `:advanced-config` feature. Since you + ;; might have to use the `:settings` section to set the token, skip the check for Settings. But check it for the + ;; other sections. + (when-not (= section-name :settings) + (when-not (premium-features/has-feature? :advanced-config) + (throw (ex-info (tru "Metabase config files require a Premium token with the :advanced-config feature.") + {})))) (log/info (u/colorize :magenta (trs "Initializing {0} from config file..." section-name)) (u/emoji "🗄️")) - (config-from-file.i/initialize-section! section-name section-config)) + (advanced-config.file.i/initialize-section! section-name section-config)) (log/info (u/colorize :magenta (trs "Done initializing from file.")) (u/emoji "🗄️"))) :ok) diff --git a/enterprise/backend/src/metabase_enterprise/config_from_file/databases.clj b/enterprise/backend/src/metabase_enterprise/advanced_config/file/databases.clj similarity index 68% rename from enterprise/backend/src/metabase_enterprise/config_from_file/databases.clj rename to enterprise/backend/src/metabase_enterprise/advanced_config/file/databases.clj index 2144b01023b5..1775f515b733 100644 --- a/enterprise/backend/src/metabase_enterprise/config_from_file/databases.clj +++ b/enterprise/backend/src/metabase_enterprise/advanced_config/file/databases.clj @@ -1,8 +1,8 @@ -(ns metabase-enterprise.config-from-file.databases +(ns metabase-enterprise.advanced-config.file.databases (:require [clojure.spec.alpha :as s] [clojure.tools.logging :as log] - [metabase-enterprise.config-from-file.interface :as config-from-file.i] + [metabase-enterprise.advanced-config.file.interface :as advanced-config.file.i] [metabase.driver.util :as driver.u] [metabase.models.database :refer [Database]] [metabase.models.setting :refer [defsetting]] @@ -17,21 +17,21 @@ :type :boolean :default true) -(s/def :metabase-enterprise.config-from-file.databases.config-file-spec/name +(s/def :metabase-enterprise.advanced-config.file.databases.config-file-spec/name string?) -(s/def :metabase-enterprise.config-from-file.databases.config-file-spec/engine +(s/def :metabase-enterprise.advanced-config.file.databases.config-file-spec/engine string?) -(s/def :metabase-enterprise.config-from-file.databases.config-file-spec/details +(s/def :metabase-enterprise.advanced-config.file.databases.config-file-spec/details map?) (s/def ::config-file-spec - (s/keys :req-un [:metabase-enterprise.config-from-file.databases.config-file-spec/engine - :metabase-enterprise.config-from-file.databases.config-file-spec/name - :metabase-enterprise.config-from-file.databases.config-file-spec/details])) + (s/keys :req-un [:metabase-enterprise.advanced-config.file.databases.config-file-spec/engine + :metabase-enterprise.advanced-config.file.databases.config-file-spec/name + :metabase-enterprise.advanced-config.file.databases.config-file-spec/details])) -(defmethod config-from-file.i/section-spec :databases +(defmethod advanced-config.file.i/section-spec :databases [_section] (s/spec (s/* ::config-file-spec))) @@ -50,7 +50,7 @@ ((requiring-resolve 'metabase.sync/sync-database!) db) (log/info (trs "Sync on database creation when initializing from file is disabled. Skipping sync."))))))) -(defmethod config-from-file.i/initialize-section! :databases +(defmethod advanced-config.file.i/initialize-section! :databases [_section-name databases] (doseq [database databases] (init-from-config-file! database))) diff --git a/enterprise/backend/src/metabase_enterprise/config_from_file/interface.clj b/enterprise/backend/src/metabase_enterprise/advanced_config/file/interface.clj similarity index 96% rename from enterprise/backend/src/metabase_enterprise/config_from_file/interface.clj rename to enterprise/backend/src/metabase_enterprise/advanced_config/file/interface.clj index 9fa276212401..ba8873d941ee 100644 --- a/enterprise/backend/src/metabase_enterprise/config_from_file/interface.clj +++ b/enterprise/backend/src/metabase_enterprise/advanced_config/file/interface.clj @@ -1,4 +1,4 @@ -(ns metabase-enterprise.config-from-file.interface +(ns metabase-enterprise.advanced-config.file.interface (:require [clojure.tools.logging :as log] [metabase.util :as u] diff --git a/enterprise/backend/src/metabase_enterprise/config_from_file/settings.clj b/enterprise/backend/src/metabase_enterprise/advanced_config/file/settings.clj similarity index 63% rename from enterprise/backend/src/metabase_enterprise/config_from_file/settings.clj rename to enterprise/backend/src/metabase_enterprise/advanced_config/file/settings.clj index ea6052d697fb..5415db2d6ef6 100644 --- a/enterprise/backend/src/metabase_enterprise/config_from_file/settings.clj +++ b/enterprise/backend/src/metabase_enterprise/advanced_config/file/settings.clj @@ -1,16 +1,16 @@ -(ns metabase-enterprise.config-from-file.settings +(ns metabase-enterprise.advanced-config.file.settings (:require [clojure.spec.alpha :as s] [clojure.tools.logging :as log] - [metabase-enterprise.config-from-file.interface :as config-from-file.i] + [metabase-enterprise.advanced-config.file.interface :as advanced-config.file.i] [metabase.models.setting :as setting] [metabase.util.i18n :refer [trs]])) -(defmethod config-from-file.i/section-spec :settings +(defmethod advanced-config.file.i/section-spec :settings [_section-name] (s/map-of keyword? any?)) -(defmethod config-from-file.i/initialize-section! :settings +(defmethod advanced-config.file.i/initialize-section! :settings [_section-name settings] (log/info (trs "Setting setting values from config file")) (doseq [[setting-name setting-value] settings] diff --git a/enterprise/backend/src/metabase_enterprise/config_from_file/users.clj b/enterprise/backend/src/metabase_enterprise/advanced_config/file/users.clj similarity index 64% rename from enterprise/backend/src/metabase_enterprise/config_from_file/users.clj rename to enterprise/backend/src/metabase_enterprise/advanced_config/file/users.clj index a2fe138ab3ca..c1c36c53bef4 100644 --- a/enterprise/backend/src/metabase_enterprise/config_from_file/users.clj +++ b/enterprise/backend/src/metabase_enterprise/advanced_config/file/users.clj @@ -1,32 +1,32 @@ -(ns metabase-enterprise.config-from-file.users +(ns metabase-enterprise.advanced-config.file.users (:require [clojure.spec.alpha :as s] [clojure.tools.logging :as log] - [metabase-enterprise.config-from-file.interface :as config-from-file.i] + [metabase-enterprise.advanced-config.file.interface :as advanced-config.file.i] [metabase.models.user :refer [User]] [metabase.util :as u] [metabase.util.i18n :as i18n :refer [trs]] [toucan.db :as db])) -(s/def :metabase-enterprise.config-from-file.users.config-file-spec/first_name +(s/def :metabase-enterprise.advanced-config.file.users.config-file-spec/first_name string?) -(s/def :metabase-enterprise.config-from-file.users.config-file-spec/last_name +(s/def :metabase-enterprise.advanced-config.file.users.config-file-spec/last_name string?) -(s/def :metabase-enterprise.config-from-file.users.config-file-spec/password +(s/def :metabase-enterprise.advanced-config.file.users.config-file-spec/password string?) -(s/def :metabase-enterprise.config-from-file.users.config-file-spec/email +(s/def :metabase-enterprise.advanced-config.file.users.config-file-spec/email string?) (s/def ::config-file-spec - (s/keys :req-un [:metabase-enterprise.config-from-file.users.config-file-spec/first_name - :metabase-enterprise.config-from-file.users.config-file-spec/last_name - :metabase-enterprise.config-from-file.users.config-file-spec/password - :metabase-enterprise.config-from-file.users.config-file-spec/email])) + (s/keys :req-un [:metabase-enterprise.advanced-config.file.users.config-file-spec/first_name + :metabase-enterprise.advanced-config.file.users.config-file-spec/last_name + :metabase-enterprise.advanced-config.file.users.config-file-spec/password + :metabase-enterprise.advanced-config.file.users.config-file-spec/email])) -(defmethod config-from-file.i/section-spec :users +(defmethod advanced-config.file.i/section-spec :users [_section] (s/spec (s/* ::config-file-spec))) @@ -53,7 +53,7 @@ (pr-str (:email user))))) (db/insert! User user)))) -(defmethod config-from-file.i/initialize-section! :users +(defmethod advanced-config.file.i/initialize-section! :users [_section-name users] (doseq [user users] (init-from-config-file! user))) diff --git a/enterprise/backend/src/metabase_enterprise/sandbox/query_processor/middleware/row_level_restrictions.clj b/enterprise/backend/src/metabase_enterprise/sandbox/query_processor/middleware/row_level_restrictions.clj index c348cfe763bd..3c70c5ddb490 100644 --- a/enterprise/backend/src/metabase_enterprise/sandbox/query_processor/middleware/row_level_restrictions.clj +++ b/enterprise/backend/src/metabase_enterprise/sandbox/query_processor/middleware/row_level_restrictions.clj @@ -121,9 +121,9 @@ attr-value))) (defn- attr-remapping->parameter [login-attributes [attr-name target]] - (let [attr-value (get login-attributes attr-name ::not-found) + (let [attr-value (get login-attributes attr-name) field-base-type (target-field->base-type target)] - (when (= attr-value ::not-found) + (when (not attr-value) (throw (ex-info (tru "Query requires user attribute `{0}`" (name attr-name)) {:type qp.error-type/missing-required-parameter}))) {:type :category diff --git a/enterprise/backend/src/metabase_enterprise/serialization/serialize.clj b/enterprise/backend/src/metabase_enterprise/serialization/serialize.clj index 90335488ff8b..72f9846716d8 100644 --- a/enterprise/backend/src/metabase_enterprise/serialization/serialize.clj +++ b/enterprise/backend/src/metabase_enterprise/serialization/serialize.clj @@ -100,7 +100,7 @@ [entity] (cond-> (dissoc entity :id :creator_id :created_at :updated_at :db_id :location :dashboard_id :fields_hash :personal_owner_id :made_public_by_id :collection_id - :pulse_id :result_metadata) + :pulse_id :result_metadata :entity_id) (some #(instance? % entity) (map type [Metric Field Segment])) (dissoc :table_id))) (defmulti ^:private serialize-one diff --git a/enterprise/backend/src/metabase_enterprise/serialization/v2/backfill_ids.clj b/enterprise/backend/src/metabase_enterprise/serialization/v2/backfill_ids.clj new file mode 100644 index 000000000000..0d398c11e63a --- /dev/null +++ b/enterprise/backend/src/metabase_enterprise/serialization/v2/backfill_ids.clj @@ -0,0 +1,40 @@ +(ns metabase-enterprise.serialization.v2.backfill-ids + "Finds all models with `:entity_id` columns, scans them for anything without a blank ID, and + generates consistent entity_id based on their hashes. + + Note that cross-JVM portability is required - but that's specified for [[java.util.Random]], + so this should produce identical IDs on all platforms and JVM implementations." + (:require + [clojure.tools.logging :as log] + [metabase-enterprise.serialization.v2.models :as serdes.models] + [metabase.logger] + [metabase.models.serialization.hash :as serdes.hash] + [metabase.util :as u] + [metabase.util.i18n :refer [trs]] + [toucan.db :as db] + [toucan.models :as models])) + +(defn backfill-ids-for + "Updates all rows of a particular model to have `:entity_id` set, based on the [[serdes.hash/identity-hash]]." + [model] + (let [missing (db/select model :entity_id nil) + pk (models/primary-key model)] + (when (seq missing) + (log/info (trs "Backfilling entity_id for {0} rows of {1}" (pr-str (count missing)) (:name model))) + (doseq [entity missing + :let [hashed (serdes.hash/identity-hash entity) + eid (u/generate-nano-id hashed)]] + (db/update! model (get entity pk) :entity_id eid))))) + +(defn- has-entity-id? [model] + (:entity_id (models/properties model))) + +(defn backfill-ids + "Updates all rows of all models that are (a) serialized and (b) have `entity_id` columns to have the + `entity_id` set. If the `entity_id` is NULL, it is set based on the [[serdes.hash/identity-hash]] for that + row." + [] + (doseq [model-name (concat serdes.models/exported-models serdes.models/inlined-models) + :let [model (db/resolve-model (symbol model-name))] + :when (has-entity-id? model)] + (backfill-ids-for model))) diff --git a/enterprise/backend/src/metabase_enterprise/serialization/v2/extract.clj b/enterprise/backend/src/metabase_enterprise/serialization/v2/extract.clj index 27e411a7276f..ceab8d48c1c7 100644 --- a/enterprise/backend/src/metabase_enterprise/serialization/v2/extract.clj +++ b/enterprise/backend/src/metabase_enterprise/serialization/v2/extract.clj @@ -7,6 +7,7 @@ [clojure.string :as str] [clojure.tools.logging :as log] [medley.core :as m] + [metabase-enterprise.serialization.v2.backfill-ids :as serdes.backfill] [metabase-enterprise.serialization.v2.models :as serdes.models] [metabase.models :refer [Card Collection Dashboard DashboardCard]] [metabase.models.collection :as collection] @@ -29,7 +30,8 @@ set)] (->> (concat unowned owned) (map collection/descendant-ids) - (reduce set/union top-ids)))) + (reduce set/union top-ids) + (set/union #{nil})))) (defn extract-metabase "Extracts the appdb into a reducible stream of serializable maps, with `:serdes/meta` keys. @@ -41,6 +43,7 @@ there." [opts] (log/tracef "Extracting Metabase with options: %s" (pr-str opts)) + (serdes.backfill/backfill-ids) (let [model-pred (if (:data-model-only opts) #{"Database" "Dimension" "Field" "FieldValues" "Metric" "Segment" "Table"} (constantly true)) @@ -166,14 +169,19 @@ serialized output." [{:keys [selected-collections targets] :as opts}] (log/tracef "Extracting subtrees with options: %s" (pr-str opts)) - (if-let [analysis (escape-analysis selected-collections)] - ;; If that is non-nil, emit the report. - (escape-report analysis) - ;; If it's nil, there are no errors, and we can proceed to do the dump. - (let [closure (descendants-closure opts targets) - by-model (->> closure - (group-by first) - (m/map-vals #(set (map second %))))] - (eduction cat (for [[model ids] by-model] - (eduction (map #(serdes.base/extract-one model opts %)) - (db/select-reducible (symbol model) :id [:in ids]))))))) + (let [selected-collections (or selected-collections (->> targets + (filter #(= (first %) "Collection")) + (map second) + set))] + (serdes.backfill/backfill-ids) + (if-let [analysis (escape-analysis selected-collections)] + ;; If that is non-nil, emit the report. + (escape-report analysis) + ;; If it's nil, there are no errors, and we can proceed to do the dump. + (let [closure (descendants-closure opts targets) + by-model (->> closure + (group-by first) + (m/map-vals #(set (map second %))))] + (eduction cat (for [[model ids] by-model] + (eduction (map #(serdes.base/extract-one model opts %)) + (db/select-reducible (symbol model) :id [:in ids])))))))) diff --git a/enterprise/backend/src/metabase_enterprise/serialization/v2/ingest/yaml.clj b/enterprise/backend/src/metabase_enterprise/serialization/v2/ingest/yaml.clj index 39d42add7937..4260eefef1f5 100644 --- a/enterprise/backend/src/metabase_enterprise/serialization/v2/ingest/yaml.clj +++ b/enterprise/backend/src/metabase_enterprise/serialization/v2/ingest/yaml.clj @@ -4,7 +4,6 @@ (:require [clojure.java.io :as io] [medley.core :as m] [metabase-enterprise.serialization.v2.ingest :as ingest] - [metabase-enterprise.serialization.v2.models :as models] [metabase-enterprise.serialization.v2.utils.yaml :as u.yaml] [metabase.util.date-2 :as u.date] [yaml.core :as yaml] @@ -22,7 +21,6 @@ ; We return a path of 1 item, the setting itself. [{:model "Setting" :id (name k)}]))) - (defn- build-metas [^File root-dir ^File file] (let [path-parts (u.yaml/path-split root-dir file)] (if (= ["settings.yaml"] path-parts) @@ -46,6 +44,9 @@ (sequential? obj) (mapv keywords obj) :else obj)) +(defn- strip-labels [hierarchy] + (mapv #(dissoc % :label) hierarchy)) + (defn- ingest-entity "Given a hierarchy, read in the YAML file it identifies. Clean it up (eg. parsing timestamps) and attach the hierarchy as `:serdes/meta`. @@ -54,42 +55,53 @@ The labels are removed from the hierarchy attached at `:serdes/meta`, since the storage system might have damaged the original labels by eg. truncating them to keep the file names from getting too long. The labels aren't used at all on the loading side, so it's fine to drop them." - [root-dir hierarchy] - (let [unlabeled (mapv #(dissoc % :label) hierarchy) - file (u.yaml/hierarchy->file root-dir hierarchy)] ; Use the original hierarchy for the filesystem. - (-> (when (.exists file) file) ; If the returned file doesn't actually exist, replace it with nil. + [hierarchy ^File file] + (-> (when (.exists file) file) ; If the returned file doesn't actually exist, replace it with nil. + + ;; No automatic keywords; it's too generous with what counts as a keyword and has a bug. + ;; See https://github.com/clj-commons/clj-yaml/issues/64 + (yaml/from-file false) + keywords + read-timestamps + (assoc :serdes/meta (strip-labels hierarchy)))) ; But return the hierarchy without labels. + +(def ^:private legal-top-level-paths + "These are all the legal first segments of paths. This is used by ingestion to avoid `.git`, `.github`, `README.md` + and other such extras." + #{"collections" "databases" "snippets" "settings.yaml"}) - ;; No automatic keywords; it's too generous with what counts as a keyword and has a bug. - ;; See https://github.com/clj-commons/clj-yaml/issues/64 - (yaml/from-file false) - keywords - read-timestamps - (assoc :serdes/meta unlabeled)))) ; But return the hierarchy without labels. +(defn- ingest-all [^File root-dir] + ;; This returns a map {unlabeled-hierarchy [original-hierarchy File]}. + (into {} (for [^File file (file-seq root-dir) + :when (and (.isFile file) + (let [rel (.relativize (.toPath root-dir) (.toPath file))] + (-> rel (.subpath 0 1) (.toString) legal-top-level-paths))) + hierarchy (build-metas root-dir file)] + [(strip-labels hierarchy) [hierarchy file]]))) -(deftype YamlIngestion [^File root-dir settings] +(deftype YamlIngestion [^File root-dir settings cache] ingest/Ingestable (ingest-list [_] - (let [model-set (set models/exported-models)] - (eduction (comp (filter (fn [^File f] (.isFile f))) - ;; The immediate parent directory should be a recognized model name. - ;; If it's not, this may be in .git, or .github/actions/... or similar extra files. - (filter (fn [^File f] (or (= (.getName f) "settings.yaml") - (-> f - (.getParentFile) - (.getName) - model-set)))) - (mapcat (partial build-metas root-dir))) - (file-seq root-dir)))) + (->> (or @cache + (reset! cache (ingest-all root-dir))) + vals + (map first))) (ingest-one [_ abs-path] + (when-not @cache + (reset! cache (ingest-all root-dir))) (let [{:keys [model id]} (first abs-path)] (if (and (= (count abs-path) 1) (= model "Setting")) {:serdes/meta abs-path :key (keyword id) :value (get settings (keyword id))} - (ingest-entity root-dir abs-path))))) + (->> abs-path + strip-labels + (get @cache) + second + (ingest-entity abs-path)))))) (defn ingest-yaml "Creates a new Ingestable on a directory of YAML files, as created by [[metabase-enterprise.serialization.v2.storage.yaml]]." [root-dir] - (->YamlIngestion (io/file root-dir) (yaml/from-file (io/file root-dir "settings.yaml")))) + (->YamlIngestion (io/file root-dir) (yaml/from-file (io/file root-dir "settings.yaml")) (atom nil))) diff --git a/enterprise/backend/src/metabase_enterprise/serialization/v2/load.clj b/enterprise/backend/src/metabase_enterprise/serialization/v2/load.clj index 8947113c5631..55c0b7f83da4 100644 --- a/enterprise/backend/src/metabase_enterprise/serialization/v2/load.clj +++ b/enterprise/backend/src/metabase_enterprise/serialization/v2/load.clj @@ -2,17 +2,30 @@ "Loading is the interesting part of deserialization: integrating the maps \"ingested\" from files into the appdb. See the detailed breakdown of the (de)serialization processes in [[metabase.models.serialization.base]]." (:require [medley.core :as m] + [metabase-enterprise.serialization.v2.backfill-ids :as serdes.backfill] [metabase-enterprise.serialization.v2.ingest :as serdes.ingest] [metabase.models.serialization.base :as serdes.base])) (declare load-one) (defn- load-deps - "Given a list of `deps` (hierarchies), `load-one` them all." + "Given a list of `deps` (hierarchies), [[load-one]] them all. + If [[load-one]] throws because it can't find that entity in the filesystem, check if it's already loaded in + our database." [ctx deps] (if (empty? deps) ctx - (reduce load-one ctx deps))) + (letfn [(loader [ctx dep] + (try + (load-one ctx dep) + (catch Exception e + (if (and (= (:error (ex-data e)) ::not-found) + (serdes.base/load-find-local dep)) + ;; It was missing but we found it locally, so just return the context. + ctx + ;; Different error, or couldn't find it locally, so rethrow. + (throw e)))))] + (reduce loader ctx deps)))) (defn- load-one "Loads a single entity, specified by its `:serdes/meta` abstract path, into the appdb, doing some bookkeeping to avoid @@ -34,7 +47,8 @@ (catch Exception e (throw (ex-info (format "Failed to read file for %s" (pr-str path)) {:path path - :deps-chain expanding} + :deps-chain expanding + :error ::not-found} e)))) deps (serdes.base/serdes-dependencies ingested) ctx (-> ctx @@ -44,9 +58,9 @@ (update :expanding disj path)) ;; Use the abstract path as attached by the ingestion process, not the original one we were passed. rebuilt-path (serdes.base/serdes-path ingested) - local-pk-or-nil (serdes.base/load-find-local rebuilt-path)] + local-or-nil (serdes.base/load-find-local rebuilt-path)] (try - (serdes.base/load-one! ingested local-pk-or-nil) + (serdes.base/load-one! ingested local-or-nil) ctx (catch Exception e (throw (ex-info (format "Failed to load into database for %s" (pr-str path)) @@ -59,6 +73,7 @@ [ingestion] ;; We proceed in the arbitrary order of ingest-list, deserializing all the files. Their declared dependencies guide ;; the import, and make sure all containers are imported before contents, etc. + (serdes.backfill/backfill-ids) (let [contents (serdes.ingest/ingest-list ingestion)] (reduce load-one {:expanding #{} :seen #{} diff --git a/enterprise/backend/src/metabase_enterprise/serialization/v2/models.clj b/enterprise/backend/src/metabase_enterprise/serialization/v2/models.clj index b1da6cc9d586..c5caad31a028 100644 --- a/enterprise/backend/src/metabase_enterprise/serialization/v2/models.clj +++ b/enterprise/backend/src/metabase_enterprise/serialization/v2/models.clj @@ -5,18 +5,20 @@ ["Card" "Collection" "Dashboard" - "DashboardCard" "Database" - "Dimension" "Field" "FieldValues" "Metric" "NativeQuerySnippet" - "Pulse" - "PulseCard" - "PulseChannel" "Segment" "Setting" "Table" - "Timeline" + "Timeline"]) + +(def inlined-models + "An additional list of models which are inlined into parent entities for serialization. + These are not extracted and serialized separately, but they may need some processing done. + For example, the models should also have their entity_id fields populated (if they have one)." + ["DashboardCard" + "Dimension" "TimelineEvent"]) diff --git a/enterprise/backend/src/metabase_enterprise/serialization/v2/storage/yaml.clj b/enterprise/backend/src/metabase_enterprise/serialization/v2/storage/yaml.clj index cb4aa7423385..27214a6197d2 100644 --- a/enterprise/backend/src/metabase_enterprise/serialization/v2/storage/yaml.clj +++ b/enterprise/backend/src/metabase_enterprise/serialization/v2/storage/yaml.clj @@ -22,8 +22,8 @@ (io/make-parents file) (spit (io/file file) (generate-yaml obj))) -(defn- store-entity! [{:keys [root-dir]} entity] - (spit-yaml (u.yaml/hierarchy->file root-dir (serdes.base/serdes-path entity)) +(defn- store-entity! [opts entity] + (spit-yaml (u.yaml/hierarchy->file opts entity) (dissoc entity :serdes/meta))) (defn- store-settings! [{:keys [root-dir]} settings] @@ -37,7 +37,8 @@ (instance? java.io.File (:root-dir opts))) (throw (ex-info ":yaml storage requires the :root-dir option to be a string or File" {:opts opts}))) - (let [settings (atom [])] + (let [settings (atom []) + opts (merge opts (serdes.base/storage-base-context))] (doseq [entity stream] (if (-> entity :serdes/meta last :model (= "Setting")) (swap! settings conj entity) diff --git a/enterprise/backend/src/metabase_enterprise/serialization/v2/utils/yaml.clj b/enterprise/backend/src/metabase_enterprise/serialization/v2/utils/yaml.clj index f40b6dc795e1..6b89fb1e22e3 100644 --- a/enterprise/backend/src/metabase_enterprise/serialization/v2/utils/yaml.clj +++ b/enterprise/backend/src/metabase_enterprise/serialization/v2/utils/yaml.clj @@ -1,24 +1,10 @@ (ns metabase-enterprise.serialization.v2.utils.yaml (:require [clojure.java.io :as io] - [clojure.string :as str]) + [clojure.string :as str] + [metabase.models.serialization.base :as serdes.base]) (:import java.io.File java.nio.file.Path)) -(def ^:private max-label-length 100) - -(defn- truncate-label [s] - (if (> (count s) max-label-length) - (subs s 0 max-label-length) - s)) - -(defn- leaf-file-name - ([id] (str id ".yaml")) - ;; + is a legal, unescaped character on all common filesystems, - ;; but doesn't appear in `identity-hash` or NanoID! - ([id label] (if (nil? label) - (leaf-file-name id) - (str id "+" (truncate-label label) ".yaml")))) - (defn- escape-segment "Given a path segment, which is supposed to be the name of a single file or directory, escape any slashes inside it. This occurs in practice, for example with a `Field.name` containing a slash like \"Company/organization website\"." @@ -35,26 +21,14 @@ (str/replace "__BACKSLASH__" "\\"))) (defn hierarchy->file - "Given a :serdes/meta abstract path, return a [[File]] corresponding to it." - ^File [root-dir hierarchy] - (let [;; All earlier parts of the hierarchy form Model/id/ pairs. - prefix (apply concat (for [{:keys [model id]} (drop-last hierarchy)] - [model id])) - ;; The last part of the hierarchy is used for the basename; this is the only part with the label. - {:keys [id model label]} (last hierarchy) - leaf-name (leaf-file-name id label) - as-given (apply io/file root-dir (map escape-segment (concat prefix [model leaf-name])))] - (if (.exists ^File as-given) - as-given - ; If that file name doesn't exist, check the directory to see if there's one that's the requested file plus a - ; human-readable portion. - (let [dir (apply io/file root-dir (map escape-segment (concat prefix [model]))) - matches (filter #(and (.startsWith ^String % (str id "+")) - (.endsWith ^String % ".yaml")) - (.list ^File dir))] - (if (empty? matches) - (io/file dir (escape-segment leaf-name)) - (io/file dir (first matches))))))) + "Given an extracted entity, return a [[File]] corresponding to it." + ^File [ctx entity] + (let [;; Get the desired [[serdes.base/storage-path]]. + base-path (serdes.base/storage-path entity ctx) + dirnames (drop-last base-path) + ;; Attach the file extension to the last part. + basename (str (last base-path) ".yaml")] + (apply io/file (:root-dir ctx) (map escape-segment (concat dirnames [basename]))))) (defn path-split "Given a root directory and a file underneath it, return a sequence of path parts to get there. @@ -67,13 +41,16 @@ (defn path->hierarchy "Given the list of file path chunks as returned by [[path-split]], reconstruct the `:serdes/meta` abstract path corresponding to it. - Note that the __SLASH__ and __BACKSLASH__ interpolations of [[escape-segment]] are reversed here." + Note that the __SLASH__ and __BACKSLASH__ interpolations of [[escape-segment]] are reversed here, and also the + file extension is stripped off the last segment. + + The heavy lifting is done by the matcher functions registered by each model using + [[serdes.base/register-ingestion-path!]]." [path-parts] - (let [parentage (into [] (for [[model id] (partition 2 (drop-last 2 path-parts))] - {:model model :id (unescape-segment id)})) - [model basename] (take-last 2 path-parts) - basename (unescape-segment basename) - [_ id label] (or (re-matches #"^([A-Za-z0-9_\.:-]+)(?:\+(.*))?\.yaml$" basename) - (re-matches #"^(.+)\.yaml$" basename))] - (conj parentage (cond-> {:model model :id id} - label (assoc :label label))))) + (let [basename (last path-parts) + basename (if (str/ends-with? basename ".yaml") + (subs basename 0 (- (count basename) 5)) + basename) + path-parts (concat (map unescape-segment (drop-last path-parts)) + [(unescape-segment basename)])] + (serdes.base/ingest-path path-parts))) diff --git a/enterprise/backend/test/metabase_enterprise/config_from_file/databases_test.clj b/enterprise/backend/test/metabase_enterprise/advanced_config/file/databases_test.clj similarity index 50% rename from enterprise/backend/test/metabase_enterprise/config_from_file/databases_test.clj rename to enterprise/backend/test/metabase_enterprise/advanced_config/file/databases_test.clj index 81c384707708..0aad0264c2a3 100644 --- a/enterprise/backend/test/metabase_enterprise/config_from_file/databases_test.clj +++ b/enterprise/backend/test/metabase_enterprise/advanced_config/file/databases_test.clj @@ -1,14 +1,19 @@ -(ns metabase-enterprise.config-from-file.databases-test +(ns metabase-enterprise.advanced-config.file.databases-test (:require [clojure.test :refer :all] - [flatland.ordered.map :as ordered-map] - [metabase-enterprise.config-from-file.core :as config-from-file] + [metabase-enterprise.advanced-config.file :as advanced-config.file] [metabase.db.connection :as mdb.connection] [metabase.models :refer [Database Table]] + [metabase.public-settings.premium-features-test :as premium-features-test] [metabase.test :as mt] [metabase.util :as u] [toucan.db :as db])) +(use-fixtures :each (fn [thunk] + (binding [advanced-config.file/*supported-versions* {:min 1, :max 1}] + (premium-features-test/with-premium-features #{:advanced-config} + (thunk))))) + (def ^:private test-db-name (u/qualified-name ::test-db)) (deftest init-from-config-file-test @@ -16,14 +21,13 @@ (let [db-type (mdb.connection/db-type) original-db (mt/with-driver db-type (mt/db))] (try - (binding [config-from-file/*supported-versions* {:min 1, :max 1} - config-from-file/*config* {:version 1 - :config {:databases [{:name test-db-name - :engine (name db-type) - :details (:details original-db)}]}}] + (binding [advanced-config.file/*config* {:version 1 + :config {:databases [{:name test-db-name + :engine (name db-type) + :details (:details original-db)}]}}] (testing "Create a Database if it does not already exist" (is (= :ok - (config-from-file/initialize!))) + (advanced-config.file/initialize!))) (let [db (db/select-one Database :name test-db-name)] (is (partial= {:engine db-type} db)) @@ -31,7 +35,7 @@ (db/count Database :name test-db-name))) (testing "do not duplicate if Database already exists" (is (= :ok - (config-from-file/initialize!))) + (advanced-config.file/initialize!))) (is (= 1 (db/count Database :name test-db-name))) (is (partial= {:engine db-type} @@ -42,44 +46,30 @@ (finally (db/delete! Database :name test-db-name)))))) -(deftest ^:parallel init-from-config-file-connection-validation-test +(deftest init-from-config-file-connection-validation-test (testing "Validate connection details when creating a Database from a config file, and error if they are invalid" - (binding [config-from-file/*supported-versions* {:min 1, :max 1} - config-from-file/*config* {:version 1 - :config {:databases [{:name (str test-db-name "-in-memory") - :engine "h2" - :details {:db "mem:some-in-memory-db"}}]}}] + (binding [advanced-config.file/*config* {:version 1 + :config {:databases [{:name (str test-db-name "-in-memory") + :engine "h2" + :details {:db "mem:some-in-memory-db"}}]}}] (is (thrown-with-msg? clojure.lang.ExceptionInfo #"Database cannot be found\." - (config-from-file/initialize!)))))) + (advanced-config.file/initialize!)))))) (deftest disable-sync-test (testing "We should be able to disable sync for new Databases by specifying a Setting in the config file" ;; make sure we're actually testing something if it was already set to false locally. (mt/with-temporary-setting-values [config-from-file-sync-databases true] (try - (binding [config-from-file/*supported-versions* {:min 1, :max 1} - config-from-file/*config* {:version 1 - :config - ;; `settings:` HAS to come before `databases:`, otherwise the - ;; flag won't be set when database sync stuff happens. - ;; - ;; Using a [[flatland.ordered.map]] here really isn't necessary - ;; since this map only has two keys and will be created as an - ;; `ArrayMap`, preserving the originally specified order... but - ;; using [[ordered-map]] explicitly here makes this constraint - ;; clearer I think. Also the YAML library reads stuff in as an - ;; ordered map so this more closely matches the behavior when - ;; using a file - (ordered-map/ordered-map - :settings {:config-from-file-sync-databases false} - :databases [{:name test-db-name - :engine "h2" - :details (:details (mt/db))}])}] + (binding [advanced-config.file/*config* {:version 1 + :config {:settings {:config-from-file-sync-databases false} + :databases [{:name test-db-name + :engine "h2" + :details (:details (mt/db))}]}}] (testing "Create a Database since it does not already exist" (is (= :ok - (config-from-file/initialize!))) + (advanced-config.file/initialize!))) (let [db (db/select-one Database :name test-db-name)] (is (partial= {:engine :h2} db)) diff --git a/enterprise/backend/test/metabase_enterprise/advanced_config/file/settings_test.clj b/enterprise/backend/test/metabase_enterprise/advanced_config/file/settings_test.clj new file mode 100644 index 000000000000..89fe95c139f6 --- /dev/null +++ b/enterprise/backend/test/metabase_enterprise/advanced_config/file/settings_test.clj @@ -0,0 +1,44 @@ +(ns metabase-enterprise.advanced-config.file.settings-test + (:require + [clojure.test :refer :all] + [metabase-enterprise.advanced-config.file :as advanced-config.file] + [metabase.models.setting :refer [defsetting]] + [metabase.public-settings.premium-features-test :as premium-features-test])) + +(use-fixtures :each (fn [thunk] + (binding [advanced-config.file/*supported-versions* {:min 1, :max 1}] + (premium-features-test/with-premium-features #{:advanced-config} + (thunk))))) + +(defsetting config-from-file-settings-test-setting + "Internal test setting." + :visibility :internal) + +(deftest settings-test + (testing "Should be able to set settings with config-from-file" + (config-from-file-settings-test-setting! nil) + (testing "happy path" + (binding [advanced-config.file/*config* {:version 1 + :config {:settings {:config-from-file-settings-test-setting "wow"}}}] + (advanced-config.file/initialize!) + (is (= "wow" + (config-from-file-settings-test-setting))))) + (testing "Wrong value type should throw an error." + (binding [advanced-config.file/*config* {:version 1 + :config {:settings {:config-from-file-settings-test-setting 1000}}}] + + (is (thrown-with-msg? + clojure.lang.ExceptionInfo + #"Input .* does not match schema" + (advanced-config.file/initialize!))) + (testing "value should not have been updated" + (is (= "wow" + (config-from-file-settings-test-setting)))))) + (testing "Invalid Setting (does not exist)" + (binding [advanced-config.file/*config* {:version 1 + :config {:settings {:config-from-file-settings-test-setting-FAKE 1000}}}] + + (is (thrown-with-msg? + clojure.lang.ExceptionInfo + #"Unknown setting: :config-from-file-settings-test-setting-FAKE" + (advanced-config.file/initialize!))))))) diff --git a/enterprise/backend/test/metabase_enterprise/config_from_file/users_test.clj b/enterprise/backend/test/metabase_enterprise/advanced_config/file/users_test.clj similarity index 52% rename from enterprise/backend/test/metabase_enterprise/config_from_file/users_test.clj rename to enterprise/backend/test/metabase_enterprise/advanced_config/file/users_test.clj index 44524094ef1f..24f6e1902a2e 100644 --- a/enterprise/backend/test/metabase_enterprise/config_from_file/users_test.clj +++ b/enterprise/backend/test/metabase_enterprise/advanced_config/file/users_test.clj @@ -1,23 +1,28 @@ -(ns metabase-enterprise.config-from-file.users-test +(ns metabase-enterprise.advanced-config.file.users-test (:require [clojure.test :refer :all] - [metabase-enterprise.config-from-file.core :as config-from-file] - [metabase-enterprise.config-from-file.users :as config-from-file.users] + [metabase-enterprise.advanced-config.file :as advanced-config.file] + [metabase-enterprise.advanced-config.file.users :as advanced-config.file.users] [metabase.models :refer [User]] + [metabase.public-settings.premium-features-test :as premium-features-test] [metabase.util.password :as u.password] [toucan.db :as db])) +(use-fixtures :each (fn [thunk] + (binding [advanced-config.file/*supported-versions* {:min 1, :max 1}] + (premium-features-test/with-premium-features #{:advanced-config} + (thunk))))) + (deftest init-from-config-file-test (try - (binding [config-from-file/*supported-versions* {:min 1, :max 1} - config-from-file/*config* {:version 1 - :config {:users [{:first_name "Cam" - :last_name "Era" - :email "cam+config-file-test@metabase.com" - :password "2cans"}]}}] + (binding [advanced-config.file/*config* {:version 1 + :config {:users [{:first_name "Cam" + :last_name "Era" + :email "cam+config-file-test@metabase.com" + :password "2cans"}]}}] (testing "Create a User if it does not already exist" (is (= :ok - (config-from-file/initialize!))) + (advanced-config.file/initialize!))) (is (partial= {:first_name "Cam" :last_name "Era" :email "cam+config-file-test@metabase.com"} @@ -28,13 +33,13 @@ (let [hashed-password (fn [] (db/select-one-field :password User :email "cam+config-file-test@metabase.com")) salt (fn [] (db/select-one-field :password_salt User :email "cam+config-file-test@metabase.com")) original-hashed-password (hashed-password)] - (binding [config-from-file/*config* {:version 1 - :config {:users [{:first_name "Cam" - :last_name "Saul" - :email "cam+config-file-test@metabase.com" - :password "2cans"}]}}] + (binding [advanced-config.file/*config* {:version 1 + :config {:users [{:first_name "Cam" + :last_name "Saul" + :email "cam+config-file-test@metabase.com" + :password "2cans"}]}}] (is (= :ok - (config-from-file/initialize!))) + (advanced-config.file/initialize!))) (is (= 1 (db/count User :email "cam+config-file-test@metabase.com"))) (is (partial= {:first_name "Cam" @@ -56,56 +61,54 @@ (deftest init-from-config-file-force-admin-for-first-user-test (testing "If this is the first user being created, always make the user a superuser regardless of what is specified" (try - (binding [config-from-file/*supported-versions* {:min 1, :max 1}] - (testing "Create the first User" - (binding [config-from-file/*config* {:version 1 - :config {:users [{:first_name "Cam" - :last_name "Era" - :email "cam+config-file-admin-test@metabase.com" - :password "2cans" - :is_superuser false}]}}] - (with-redefs [config-from-file.users/init-from-config-file-is-first-user? (constantly true)] - (is (= :ok - (config-from-file/initialize!))) - (is (partial= {:first_name "Cam" - :last_name "Era" - :email "cam+config-file-admin-test@metabase.com" - :is_superuser true} - (db/select-one User :email "cam+config-file-admin-test@metabase.com"))) - (is (= 1 - (db/count User :email "cam+config-file-admin-test@metabase.com")))))) - (testing "Create the another User, DO NOT force them to be an admin" - (binding [config-from-file/*config* {:version 1 - :config {:users [{:first_name "Cam" - :last_name "Saul" - :email "cam+config-file-admin-test-2@metabase.com" - :password "2cans" - :is_superuser false}]}}] + (testing "Create the first User" + (binding [advanced-config.file/*config* {:version 1 + :config {:users [{:first_name "Cam" + :last_name "Era" + :email "cam+config-file-admin-test@metabase.com" + :password "2cans" + :is_superuser false}]}}] + (with-redefs [advanced-config.file.users/init-from-config-file-is-first-user? (constantly true)] (is (= :ok - (config-from-file/initialize!))) + (advanced-config.file/initialize!))) (is (partial= {:first_name "Cam" - :last_name "Saul" - :email "cam+config-file-admin-test-2@metabase.com" - :is_superuser false} - (db/select-one User :email "cam+config-file-admin-test-2@metabase.com"))) + :last_name "Era" + :email "cam+config-file-admin-test@metabase.com" + :is_superuser true} + (db/select-one User :email "cam+config-file-admin-test@metabase.com"))) (is (= 1 - (db/count User :email "cam+config-file-admin-test-2@metabase.com")))))) + (db/count User :email "cam+config-file-admin-test@metabase.com")))))) + (testing "Create the another User, DO NOT force them to be an admin" + (binding [advanced-config.file/*config* {:version 1 + :config {:users [{:first_name "Cam" + :last_name "Saul" + :email "cam+config-file-admin-test-2@metabase.com" + :password "2cans" + :is_superuser false}]}}] + (is (= :ok + (advanced-config.file/initialize!))) + (is (partial= {:first_name "Cam" + :last_name "Saul" + :email "cam+config-file-admin-test-2@metabase.com" + :is_superuser false} + (db/select-one User :email "cam+config-file-admin-test-2@metabase.com"))) + (is (= 1 + (db/count User :email "cam+config-file-admin-test-2@metabase.com"))))) (finally (db/delete! User :email [:in #{"cam+config-file-admin-test@metabase.com" "cam+config-file-admin-test-2@metabase.com"}]))))) (deftest init-from-config-file-env-var-for-password-test (testing "Ensure that we can set User password using {{env ...}} templates" (try - (binding [config-from-file/*supported-versions* {:min 1, :max 1} - config-from-file/*config* {:version 1 - :config {:users [{:first_name "Cam" - :last_name "Era" - :email "cam+config-file-password-test@metabase.com" - :password "{{env USER_PASSWORD}}"}]}} - config-from-file/*env* (assoc @#'config-from-file/*env* :user-password "1234cans")] + (binding [advanced-config.file/*config* {:version 1 + :config {:users [{:first_name "Cam" + :last_name "Era" + :email "cam+config-file-password-test@metabase.com" + :password "{{env USER_PASSWORD}}"}]}} + advanced-config.file/*env* (assoc @#'advanced-config.file/*env* :user-password "1234cans")] (testing "Create a User if it does not already exist" (is (= :ok - (config-from-file/initialize!))) + (advanced-config.file/initialize!))) (let [user (db/select-one [User :first_name :last_name :email :password_salt :password] :email "cam+config-file-password-test@metabase.com")] (is (partial= {:first_name "Cam" @@ -116,34 +119,33 @@ (finally (db/delete! User :email "cam+config-file-password-test@metabase.com"))))) -(deftest ^:parallel init-from-config-file-validation-test - (binding [config-from-file/*supported-versions* {:min 1, :max 1}] - (are [user error-pattern] (thrown-with-msg? - clojure.lang.ExceptionInfo - error-pattern - (binding [config-from-file/*config* {:version 1 - :config {:users [user]}}] - (#'config-from-file/config))) - ;; missing email - {:first_name "Cam" - :last_name "Era" - :password "2cans"} - (re-pattern (java.util.regex.Pattern/quote "failed: (contains? % :email)")) +(deftest init-from-config-file-validation-test + (are [user error-pattern] (thrown-with-msg? + clojure.lang.ExceptionInfo + error-pattern + (binding [advanced-config.file/*config* {:version 1 + :config {:users [user]}}] + (#'advanced-config.file/config))) + ;; missing email + {:first_name "Cam" + :last_name "Era" + :password "2cans"} + (re-pattern (java.util.regex.Pattern/quote "failed: (contains? % :email)")) - ;; missing first name - {:last_name "Era" - :email "cam+config-file-admin-test@metabase.com" - :password "2cans"} - (re-pattern (java.util.regex.Pattern/quote "failed: (contains? % :first_name)")) + ;; missing first name + {:last_name "Era" + :email "cam+config-file-admin-test@metabase.com" + :password "2cans"} + (re-pattern (java.util.regex.Pattern/quote "failed: (contains? % :first_name)")) - ;; missing last name - {:first_name "Cam" - :email "cam+config-file-admin-test@metabase.com" - :password "2cans"} - (re-pattern (java.util.regex.Pattern/quote "failed: (contains? % :last_name)")) + ;; missing last name + {:first_name "Cam" + :email "cam+config-file-admin-test@metabase.com" + :password "2cans"} + (re-pattern (java.util.regex.Pattern/quote "failed: (contains? % :last_name)")) - ;; missing password - {:first_name "Cam" - :last_name "Era" - :email "cam+config-file-test@metabase.com"} - (re-pattern (java.util.regex.Pattern/quote "failed: (contains? % :password)"))))) + ;; missing password + {:first_name "Cam" + :last_name "Era" + :email "cam+config-file-test@metabase.com"} + (re-pattern (java.util.regex.Pattern/quote "failed: (contains? % :password)")))) diff --git a/enterprise/backend/test/metabase_enterprise/advanced_config/file_test.clj b/enterprise/backend/test/metabase_enterprise/advanced_config/file_test.clj new file mode 100644 index 000000000000..201aa13cf55b --- /dev/null +++ b/enterprise/backend/test/metabase_enterprise/advanced_config/file_test.clj @@ -0,0 +1,236 @@ +(ns metabase-enterprise.advanced-config.file-test + (:require + [clojure.string :as str] + [clojure.test :refer :all] + [clojure.walk :as walk] + [metabase-enterprise.advanced-config.file :as advanced-config.file] + [metabase-enterprise.advanced-config.file.interface :as advanced-config.file.i] + [metabase.public-settings.premium-features-test :as premium-features-test] + [metabase.test :as mt] + [metabase.util :as u] + [yaml.core :as yaml])) + +(use-fixtures :each (fn [thunk] + (binding [advanced-config.file/*supported-versions* {:min 1.0, :max 1.999}] + (premium-features-test/with-premium-features #{:advanced-config} + (thunk))))) + +(defn- re-quote [^String s] + (re-pattern (java.util.regex.Pattern/quote s))) + +(def ^:private mock-yaml + {:version 1 + :config {:settings {:my-setting "abc123"}}}) + +(deftest config-test + (testing "Specify a custom path and read from YAML" + (mt/with-temp-file [filename "temp-config-file.yml"] + (spit filename (yaml/generate-string mock-yaml)) + (binding [advanced-config.file/*env* (assoc @#'advanced-config.file/*env* :mb-config-file-path filename)] + (is (= {:version 1 + :config {:settings {:my-setting "abc123"}}} + (#'advanced-config.file/config)))))) + (testing "Support overriding config with dynamic var for mocking purposes" + (binding [advanced-config.file/*config* mock-yaml] + (is (= {:version 1 + :config {:settings {:my-setting "abc123"}}} + (#'advanced-config.file/config)))))) + +(deftest validate-config-test + (testing "Config should throw an error" + (testing "if it is not a map" + (binding [advanced-config.file/*config* [1 2 3 4]] + (is (thrown-with-msg? + clojure.lang.ExceptionInfo + (re-quote "failed: map?") + (#'advanced-config.file/config))))) + (testing "if version" + (testing "is not included" + (binding [advanced-config.file/*config* {:config {:settings {}}}] + (is (thrown-with-msg? + clojure.lang.ExceptionInfo + (re-quote "failed: (contains? % :version)") + (#'advanced-config.file/config))))) + (testing "is unsupported" + (testing "because it is too old" + (binding [advanced-config.file/*supported-versions* {:min 2.0, :max 3.0} + advanced-config.file/*config* {:version 1.0, :config {}}] + (is (thrown-with-msg? + clojure.lang.ExceptionInfo + (re-quote "failed: supported-version?") + (#'advanced-config.file/config))))) + (testing "because it is too new" + (binding [advanced-config.file/*supported-versions* {:min 2.0, :max 3.0} + advanced-config.file/*config* {:version 4.0, :config {}}] + (is (thrown-with-msg? + clojure.lang.ExceptionInfo + (re-quote "failed: supported-version?") + (#'advanced-config.file/config))))))))) + +(defn- mock-config-with-setting [s] + {:version 1.0, :config {:settings {:my-setting s}}}) + +(deftest expand-template-forms-test + (testing "Ignore single curly brackets, or brackets with spaces between them" + (are [s] (= (mock-config-with-setting s) + (binding [advanced-config.file/*config* (mock-config-with-setting s)] + (#'advanced-config.file/config))) + "{}}" + "{}}" + "{ {}}")) + (testing "Invalid template forms" + (are [template error-pattern] (thrown-with-msg? + clojure.lang.ExceptionInfo + error-pattern + (binding [advanced-config.file/*config* (mock-config-with-setting template)] + (#'advanced-config.file/config))) + ;; {{ without a corresponding }} + "{{}" (re-quote "Invalid query: found [[ or {{ with no matching ]] or }}") + "{{} }" (re-quote "Invalid query: found [[ or {{ with no matching ]] or }}") + ;; raw token, not a list + "{{CONFIG_FILE_BIRD_NAME}}" (re-quote "CONFIG_FILE_BIRD_NAME - failed: valid-template-type?") + ;; unbalanced parens + "{{(env MY_ENV_VAR}}" (re-quote "Error parsing template string \"(env MY_ENV_VAR\": EOF while reading") + ;; unknown template type + "{{bird \"Parrot Hilton\"}}" (re-quote "bird - failed: valid-template-type?")))) + +(deftest recursive-template-form-expansion-test + (testing "Recursive expansion is unsupported, for now." + (binding [advanced-config.file/*env* (assoc @#'advanced-config.file/*env* :x "{{env Y}}", :y "Y") + advanced-config.file/*config* (mock-config-with-setting "{{env X}}")] + (is (= (mock-config-with-setting "{{env Y}}") + (#'advanced-config.file/config)))))) + +(deftest expand-template-env-var-values-test + (testing "env var values" + (binding [advanced-config.file/*env* (assoc @#'advanced-config.file/*env* :config-file-bird-name "Parrot Hilton")] + (testing "Nothing weird" + (binding [advanced-config.file/*config* (mock-config-with-setting "{{env CONFIG_FILE_BIRD_NAME}}")] + (is (= (mock-config-with-setting "Parrot Hilton") + (#'advanced-config.file/config))))) + (testing "Should handle multiple templates in one string" + (binding [advanced-config.file/*config* (mock-config-with-setting "{{env CONFIG_FILE_BIRD_NAME}}-{{env CONFIG_FILE_BIRD_NAME}}")] + (is (= (mock-config-with-setting "Parrot Hilton-Parrot Hilton") + (#'advanced-config.file/config))))) + (testing "Ignore whitespace inside the template brackets" + (binding [advanced-config.file/*config* (mock-config-with-setting "{{ env CONFIG_FILE_BIRD_NAME }}")] + (is (= (mock-config-with-setting "Parrot Hilton") + (#'advanced-config.file/config))))) + (testing "Ignore excess brackets" + (are [template expected] (= (mock-config-with-setting expected) + (binding [advanced-config.file/*config* (mock-config-with-setting template)] + (#'advanced-config.file/config))) + "{{{env CONFIG_FILE_BIRD_NAME}}" "{Parrot Hilton" + "{{env CONFIG_FILE_BIRD_NAME}}}" "Parrot Hilton}")) + (testing "handle lisp-case/snake-case and case variations" + (binding [advanced-config.file/*config* (mock-config-with-setting "{{env config-file-bird-name}}")] + (is (= (mock-config-with-setting "Parrot Hilton") + (#'advanced-config.file/config)))))))) + +(deftest expand-template-env-var-values-validation-test + (testing "(config) should walk the config map and expand {{template}} forms" + (testing "env var values" + (testing "validation" + (are [template error-pattern] (thrown-with-msg? + clojure.lang.ExceptionInfo + error-pattern + (binding [advanced-config.file/*config* (mock-config-with-setting template)] + (#'advanced-config.file/config))) + ;; missing env var name + "{{env}}" #"Insufficient input" + ;; too many args + "{{env SOME_ENV_VAR SOME_ENV_VAR}}" #"failed: Extra input" + ;; wrong env var type + "{{env :SOME_ENV_VAR}}" (re-quote "SOME_ENV_VAR - failed: symbol?")))))) + +(deftest optional-template-test + (testing "[[optional {{template}}]] values" + (binding [advanced-config.file/*env* (assoc @#'advanced-config.file/*env* :my-sensitive-password "~~~SeCrEt123~~~")] + (testing "env var exists" + (binding [advanced-config.file/*config* (mock-config-with-setting "[[{{env MY_SENSITIVE_PASSWORD}}]]")] + (is (= (mock-config-with-setting "~~~SeCrEt123~~~") + (#'advanced-config.file/config)))) + (binding [advanced-config.file/*config* (mock-config-with-setting "password__[[{{env MY_SENSITIVE_PASSWORD}}]]")] + (is (= (mock-config-with-setting "password__~~~SeCrEt123~~~") + (#'advanced-config.file/config)))) + (testing "with text inside optional brackets before/after the templated part" + (binding [advanced-config.file/*config* (mock-config-with-setting "[[before__{{env MY_SENSITIVE_PASSWORD}}__after]]")] + (is (= (mock-config-with-setting "before__~~~SeCrEt123~~~__after") + (#'advanced-config.file/config)))))) + (testing "env var does not exist" + (binding [advanced-config.file/*config* (mock-config-with-setting "[[{{env MY_OTHER_SENSITIVE_PASSWORD}}]]")] + (is (= (mock-config-with-setting "") + (#'advanced-config.file/config)))) + (binding [advanced-config.file/*config* (mock-config-with-setting "password__[[{{env MY_OTHER_SENSITIVE_PASSWORD}}]]")] + (is (= (mock-config-with-setting "password__") + (#'advanced-config.file/config)))) + (testing "with text inside optional brackets before/after the templated part" + (binding [advanced-config.file/*config* (mock-config-with-setting "[[before__{{env MY_OTHER_SENSITIVE_PASSWORD}}__after]]")] + (is (= (mock-config-with-setting "") + (#'advanced-config.file/config))))))))) + +(deftest initialize-section-test + (testing "Ignore unknown sections" + (binding [advanced-config.file/*config* {:version 1.0, :config {:unknown-section {}}}] + (let [log-messages (mt/with-log-messages-for-level [metabase-enterprise.advanced-config.file.interface :warn] + (is (= :ok + (advanced-config.file/initialize!))))] + (is (= [[:warn nil (u/colorize :yellow "Ignoring unknown config section :unknown-section.")]] + log-messages)))))) + +(deftest require-advanced-config-test + (testing "Config files should require the `:advanced-config` token feature" + (premium-features-test/with-premium-features #{} + (binding [advanced-config.file/*config* {:version 1.0, :config {:unknown-section {}}}] + (is (thrown-with-msg? + clojure.lang.ExceptionInfo + #"Metabase config files require a Premium token with the :advanced-config feature" + (advanced-config.file/initialize!))))))) + +(deftest error-validation-do-not-leak-env-vars-test + (testing "spec errors should not include contents of env vars -- expand templates after spec validation." + (binding [advanced-config.file/*env* (assoc @#'advanced-config.file/*env* :my-sensitive-password "~~~SeCrEt123~~~") + advanced-config.file/*config* {:version 1 + :config {:users [{:first_name "Cam" + :last_name "Era" + :password "{{env MY_SENSITIVE_PASSWORD}}"}]}}] + (is (thrown? + clojure.lang.ExceptionInfo + (#'advanced-config.file/config))) + (try + (#'advanced-config.file/config) + (catch Throwable e + (letfn [(contains-password? [form] + (let [seen-password? (atom false)] + (walk/postwalk + (fn [form] + (when (and (string? form) + (str/includes? form "~~~SeCrEt123~~~")) + (reset! seen-password? true)) + form) + form) + @seen-password?))] + (is (not (contains-password? (ex-message e)))) + (is (not (contains-password? (ex-data e)))))))))) + +(deftest always-init-settings-first-test + (testing "Always apply the :settings section first regardless of the order the YAML file is in." + (doseq [config [{:settings {:my-setting 1000} + :users [{:first_name "Cam" + :last_name "Era" + :email "camera@example.com" + :password "toucans"}]} + {:users [{:first_name "Cam" + :last_name "Era" + :email "camera@example.com" + :password "toucans"}] + :settings {:my-setting 1000}}]] + (testing (format "config = %s" (pr-str config)) + (let [initialized-sections (atom [])] + (with-redefs [advanced-config.file.i/initialize-section! (fn [section-name _section-config] + (swap! initialized-sections conj section-name))] + (binding [advanced-config.file/*config* {:version 1, :config config}] + (advanced-config.file/initialize!) + (is (= [:settings + :users] + @initialized-sections))))))))) diff --git a/enterprise/backend/test/metabase_enterprise/advanced_permissions/api/subscription_test.clj b/enterprise/backend/test/metabase_enterprise/advanced_permissions/api/subscription_test.clj index 7b529b8b02f8..500f93c79ad5 100644 --- a/enterprise/backend/test/metabase_enterprise/advanced_permissions/api/subscription_test.clj +++ b/enterprise/backend/test/metabase_enterprise/advanced_permissions/api/subscription_test.clj @@ -26,12 +26,10 @@ (deftest pulse-permissions-test (testing "/api/pulse/*" (with-subscription-disabled-for-all-users - (mt/with-user-in-groups - [group {:name "New Group"} - user [group]] - (mt/with-temp* - [Card [card] - Pulse [pulse]] + (mt/with-user-in-groups [group {:name "New Group"} + user [group]] + (mt/with-temp* [Card [card] + Pulse [pulse {:creator_id (u/the-id user)}]] (let [pulse-default {:name "A Pulse" :cards [{:id (:id card) :include_csv true @@ -81,10 +79,9 @@ user [group]] (mt/with-temp* [Card [{card-id :id}]] (letfn [(add-pulse-recipient [req-user status] - (pulse-test/with-pulse-for-card - [the-pulse - {:card card-id - :channel :email}] + (pulse-test/with-pulse-for-card [the-pulse {:card card-id + :pulse {:creator_id (u/the-id user)} + :channel :email}] (let [the-pulse (pulse/retrieve-pulse (:id the-pulse)) channel (api.alert/email-channel the-pulse) new-channel (assoc channel :recipients (conj (:recipients channel) (mt/fetch-user :lucky))) @@ -93,10 +90,9 @@ (mt/user-http-request req-user :put status (format "pulse/%d" (:id the-pulse)) new-pulse))))) (remove-pulse-recipient [req-user status] - (pulse-test/with-pulse-for-card - [the-pulse - {:card card-id - :channel :email}] + (pulse-test/with-pulse-for-card [the-pulse {:card card-id + :pulse {:creator_id (u/the-id user)} + :channel :email}] ;; manually add another user as recipient (mt/with-temp PulseChannelRecipient [_ {:user_id (:id user) :pulse_channel_id diff --git a/enterprise/backend/test/metabase_enterprise/config_from_file/core_test.clj b/enterprise/backend/test/metabase_enterprise/config_from_file/core_test.clj deleted file mode 100644 index 1cbecb6d9a9d..000000000000 --- a/enterprise/backend/test/metabase_enterprise/config_from_file/core_test.clj +++ /dev/null @@ -1,202 +0,0 @@ -(ns metabase-enterprise.config-from-file.core-test - (:require - [clojure.string :as str] - [clojure.test :refer :all] - [clojure.walk :as walk] - [metabase-enterprise.config-from-file.core :as config-from-file] - [metabase.test :as mt] - [metabase.util :as u] - [yaml.core :as yaml])) - -(use-fixtures :each (fn [thunk] - (binding [config-from-file/*supported-versions* {:min 1.0, :max 1.999}] - (thunk)))) - -(defn- re-quote [^String s] - (re-pattern (java.util.regex.Pattern/quote s))) - -(def ^:private mock-yaml - {:version 1 - :config {:settings {:my-setting "abc123"}}}) - -(deftest ^:parallel config-test - (testing "Specify a custom path and read from YAML" - (mt/with-temp-file [filename "temp-config-file.yml"] - (spit filename (yaml/generate-string mock-yaml)) - (binding [config-from-file/*env* (assoc @#'config-from-file/*env* :mb-config-file-path filename)] - (is (= {:version 1 - :config {:settings {:my-setting "abc123"}}} - (#'config-from-file/config)))))) - (testing "Support overriding config with dynamic var for mocking purposes" - (binding [config-from-file/*config* mock-yaml] - (is (= {:version 1 - :config {:settings {:my-setting "abc123"}}} - (#'config-from-file/config)))))) - -(deftest ^:parallel validate-config-test - (testing "Config should throw an error" - (testing "if it is not a map" - (binding [config-from-file/*config* [1 2 3 4]] - (is (thrown-with-msg? - clojure.lang.ExceptionInfo - (re-quote "failed: map?") - (#'config-from-file/config))))) - (testing "if version" - (testing "is not included" - (binding [config-from-file/*config* {:config {:settings {}}}] - (is (thrown-with-msg? - clojure.lang.ExceptionInfo - (re-quote "failed: (contains? % :version)") - (#'config-from-file/config))))) - (testing "is unsupported" - (testing "because it is too old" - (binding [config-from-file/*supported-versions* {:min 2.0, :max 3.0} - config-from-file/*config* {:version 1.0, :config {}}] - (is (thrown-with-msg? - clojure.lang.ExceptionInfo - (re-quote "failed: supported-version?") - (#'config-from-file/config))))) - (testing "because it is too new" - (binding [config-from-file/*supported-versions* {:min 2.0, :max 3.0} - config-from-file/*config* {:version 4.0, :config {}}] - (is (thrown-with-msg? - clojure.lang.ExceptionInfo - (re-quote "failed: supported-version?") - (#'config-from-file/config))))))))) - -(defn- mock-config-with-setting [s] - {:version 1.0, :config {:settings {:my-setting s}}}) - -(deftest ^:parallel expand-template-forms-test - (testing "Ignore single curly brackets, or brackets with spaces between them" - (are [s] (= (mock-config-with-setting s) - (binding [config-from-file/*config* (mock-config-with-setting s)] - (#'config-from-file/config))) - "{}}" - "{}}" - "{ {}}")) - (testing "Invalid template forms" - (are [template error-pattern] (thrown-with-msg? - clojure.lang.ExceptionInfo - error-pattern - (binding [config-from-file/*config* (mock-config-with-setting template)] - (#'config-from-file/config))) - ;; {{ without a corresponding }} - "{{}" (re-quote "Invalid query: found [[ or {{ with no matching ]] or }}") - "{{} }" (re-quote "Invalid query: found [[ or {{ with no matching ]] or }}") - ;; raw token, not a list - "{{CONFIG_FILE_BIRD_NAME}}" (re-quote "CONFIG_FILE_BIRD_NAME - failed: valid-template-type?") - ;; unbalanced parens - "{{(env MY_ENV_VAR}}" (re-quote "Error parsing template string \"(env MY_ENV_VAR\": EOF while reading") - ;; unknown template type - "{{bird \"Parrot Hilton\"}}" (re-quote "bird - failed: valid-template-type?")))) - -(deftest ^:parallel recursive-template-form-expansion-test - (testing "Recursive expansion is unsupported, for now." - (binding [config-from-file/*env* (assoc @#'config-from-file/*env* :x "{{env Y}}", :y "Y") - config-from-file/*config* (mock-config-with-setting "{{env X}}")] - (is (= (mock-config-with-setting "{{env Y}}") - (#'config-from-file/config)))))) - -(deftest ^:parallel expand-template-env-var-values-test - (testing "env var values" - (binding [config-from-file/*env* (assoc @#'config-from-file/*env* :config-file-bird-name "Parrot Hilton")] - (testing "Nothing weird" - (binding [config-from-file/*config* (mock-config-with-setting "{{env CONFIG_FILE_BIRD_NAME}}")] - (is (= (mock-config-with-setting "Parrot Hilton") - (#'config-from-file/config))))) - (testing "Should handle multiple templates in one string" - (binding [config-from-file/*config* (mock-config-with-setting "{{env CONFIG_FILE_BIRD_NAME}}-{{env CONFIG_FILE_BIRD_NAME}}")] - (is (= (mock-config-with-setting "Parrot Hilton-Parrot Hilton") - (#'config-from-file/config))))) - (testing "Ignore whitespace inside the template brackets" - (binding [config-from-file/*config* (mock-config-with-setting "{{ env CONFIG_FILE_BIRD_NAME }}")] - (is (= (mock-config-with-setting "Parrot Hilton") - (#'config-from-file/config))))) - (testing "Ignore excess brackets" - (are [template expected] (= (mock-config-with-setting expected) - (binding [config-from-file/*config* (mock-config-with-setting template)] - (#'config-from-file/config))) - "{{{env CONFIG_FILE_BIRD_NAME}}" "{Parrot Hilton" - "{{env CONFIG_FILE_BIRD_NAME}}}" "Parrot Hilton}")) - (testing "handle lisp-case/snake-case and case variations" - (binding [config-from-file/*config* (mock-config-with-setting "{{env config-file-bird-name}}")] - (is (= (mock-config-with-setting "Parrot Hilton") - (#'config-from-file/config)))))))) - -(deftest ^:parallel expand-template-env-var-values-validation-test - (testing "(config) should walk the config map and expand {{template}} forms" - (testing "env var values" - (testing "validation" - (are [template error-pattern] (thrown-with-msg? - clojure.lang.ExceptionInfo - error-pattern - (binding [config-from-file/*config* (mock-config-with-setting template)] - (#'config-from-file/config))) - ;; missing env var name - "{{env}}" #"Insufficient input" - ;; too many args - "{{env SOME_ENV_VAR SOME_ENV_VAR}}" #"failed: Extra input" - ;; wrong env var type - "{{env :SOME_ENV_VAR}}" (re-quote "SOME_ENV_VAR - failed: symbol?")))))) - -(deftest ^:parallel optional-template-test - (testing "[[optional {{template}}]] values" - (binding [config-from-file/*env* (assoc @#'config-from-file/*env* :my-sensitive-password "~~~SeCrEt123~~~")] - (testing "env var exists" - (binding [config-from-file/*config* (mock-config-with-setting "[[{{env MY_SENSITIVE_PASSWORD}}]]")] - (is (= (mock-config-with-setting "~~~SeCrEt123~~~") - (#'config-from-file/config)))) - (binding [config-from-file/*config* (mock-config-with-setting "password__[[{{env MY_SENSITIVE_PASSWORD}}]]")] - (is (= (mock-config-with-setting "password__~~~SeCrEt123~~~") - (#'config-from-file/config)))) - (testing "with text inside optional brackets before/after the templated part" - (binding [config-from-file/*config* (mock-config-with-setting "[[before__{{env MY_SENSITIVE_PASSWORD}}__after]]")] - (is (= (mock-config-with-setting "before__~~~SeCrEt123~~~__after") - (#'config-from-file/config)))))) - (testing "env var does not exist" - (binding [config-from-file/*config* (mock-config-with-setting "[[{{env MY_OTHER_SENSITIVE_PASSWORD}}]]")] - (is (= (mock-config-with-setting "") - (#'config-from-file/config)))) - (binding [config-from-file/*config* (mock-config-with-setting "password__[[{{env MY_OTHER_SENSITIVE_PASSWORD}}]]")] - (is (= (mock-config-with-setting "password__") - (#'config-from-file/config)))) - (testing "with text inside optional brackets before/after the templated part" - (binding [config-from-file/*config* (mock-config-with-setting "[[before__{{env MY_OTHER_SENSITIVE_PASSWORD}}__after]]")] - (is (= (mock-config-with-setting "") - (#'config-from-file/config))))))))) - -(deftest initialize-section-test - (testing "Ignore unknown sections" - (binding [config-from-file/*config* {:version 1.0, :config {:unknown-section {}}}] - (let [log-messages (mt/with-log-messages-for-level [metabase-enterprise.config-from-file.interface :warn] - (is (= :ok - (config-from-file/initialize!))))] - (is (= [[:warn nil (u/colorize :yellow "Ignoring unknown config section :unknown-section.")]] - log-messages)))))) - -(deftest ^:parallel error-validation-do-not-leak-env-vars-test - (testing "spec errors should not include contents of env vars -- expand templates after spec validation." - (binding [config-from-file/*env* (assoc @#'config-from-file/*env* :my-sensitive-password "~~~SeCrEt123~~~") - config-from-file/*config* {:version 1 - :config {:users [{:first_name "Cam" - :last_name "Era" - :password "{{env MY_SENSITIVE_PASSWORD}}"}]}}] - (is (thrown? - clojure.lang.ExceptionInfo - (#'config-from-file/config))) - (try - (#'config-from-file/config) - (catch Throwable e - (letfn [(contains-password? [form] - (let [seen-password? (atom false)] - (walk/postwalk - (fn [form] - (when (and (string? form) - (str/includes? form "~~~SeCrEt123~~~")) - (reset! seen-password? true)) - form) - form) - @seen-password?))] - (is (not (contains-password? (ex-message e)))) - (is (not (contains-password? (ex-data e)))))))))) diff --git a/enterprise/backend/test/metabase_enterprise/config_from_file/settings_test.clj b/enterprise/backend/test/metabase_enterprise/config_from_file/settings_test.clj deleted file mode 100644 index 3994c3a63807..000000000000 --- a/enterprise/backend/test/metabase_enterprise/config_from_file/settings_test.clj +++ /dev/null @@ -1,39 +0,0 @@ -(ns metabase-enterprise.config-from-file.settings-test - (:require - [clojure.test :refer :all] - [metabase-enterprise.config-from-file.core :as config-from-file] - [metabase.models.setting :refer [defsetting]])) - -(defsetting config-from-file-settings-test-setting - "Internal test setting." - :visibility :internal) - -(deftest settings-test - (testing "Should be able to set settings with config-from-file" - (config-from-file-settings-test-setting! nil) - (binding [config-from-file/*supported-versions* {:min 1, :max 1}] - (testing "happy path" - (binding [config-from-file/*config* {:version 1 - :config {:settings {:config-from-file-settings-test-setting "wow"}}}] - (config-from-file/initialize!) - (is (= "wow" - (config-from-file-settings-test-setting))))) - (testing "Wrong value type should throw an error." - (binding [config-from-file/*config* {:version 1 - :config {:settings {:config-from-file-settings-test-setting 1000}}}] - - (is (thrown-with-msg? - clojure.lang.ExceptionInfo - #"Input .* does not match schema" - (config-from-file/initialize!))) - (testing "value should not have been updated" - (is (= "wow" - (config-from-file-settings-test-setting)))))) - (testing "Invalid Setting (does not exist)" - (binding [config-from-file/*config* {:version 1 - :config {:settings {:config-from-file-settings-test-setting-FAKE 1000}}}] - - (is (thrown-with-msg? - clojure.lang.ExceptionInfo - #"Unknown setting: :config-from-file-settings-test-setting-FAKE" - (config-from-file/initialize!)))))))) diff --git a/enterprise/backend/test/metabase_enterprise/sandbox/pulse_test.clj b/enterprise/backend/test/metabase_enterprise/sandbox/pulse_test.clj index 113a21105225..6162e91d89d1 100644 --- a/enterprise/backend/test/metabase_enterprise/sandbox/pulse_test.clj +++ b/enterprise/backend/test/metabase_enterprise/sandbox/pulse_test.clj @@ -1,16 +1,20 @@ (ns metabase-enterprise.sandbox.pulse-test - (:require [clojure.data.csv :as csv] - [clojure.java.io :as io] - [clojure.test :refer :all] - [medley.core :as m] - [metabase.email.messages :as messages] - [metabase.models :refer [Card Pulse PulseCard PulseChannel PulseChannelRecipient]] - [metabase.models.pulse :as pulse] - metabase.pulse - [metabase.pulse.test-util :as pulse.tu] - [metabase.query-processor :as qp] - [metabase.test :as mt] - [metabase.util :as u])) + (:require + [clojure.data.csv :as csv] + [clojure.java.io :as io] + [clojure.test :refer :all] + [medley.core :as m] + [metabase-enterprise.sandbox.api.util :as mt.api.u] + [metabase.api.alert :as api.alert] + [metabase.email.messages :as messages] + [metabase.models + :refer [Card Pulse PulseCard PulseChannel PulseChannelRecipient]] + [metabase.models.pulse :as pulse] + [metabase.pulse] + [metabase.pulse.test-util :as pulse.tu] + [metabase.query-processor :as qp] + [metabase.test :as mt] + [metabase.util :as u])) (deftest sandboxed-pulse-test (testing "Pulses should get sent with the row-level restrictions of the User that created them." @@ -152,3 +156,65 @@ (testing "CSV attachment" (is (= 23 (csv->row-count attachment)))))))))))) + +(deftest sandboxed-users-cant-read-pulse-recipients + (testing "When sandboxed users fetch a pulse hydrated with recipients, they should only see themselves" + (mt/with-temp* [Pulse [{pulse-id :id} {:name "my pulse"}] + PulseChannel [{pc-id :id} {:pulse_id pulse-id + :channel_type :email}] + PulseChannelRecipient [_ {:pulse_channel_id pc-id, :user_id (mt/user->id :crowberto)}] + PulseChannelRecipient [_ {:pulse_channel_id pc-id, :user_id (mt/user->id :rasta)}]] + (let [recipient-ids (fn [pulses] + (let [pulse (first (filter #(= pulse-id (:id %)) pulses)) + recipients (-> pulse :channels first :recipients)] + (sort (map :id recipients))))] + (mt/with-test-user :rasta + (with-redefs [mt.api.u/segmented-user? (constantly false)] + (is (= (sort [(mt/user->id :rasta) (mt/user->id :crowberto)]) + (-> (mt/user-http-request :rasta :get 200 "pulse/") + recipient-ids))) + + (is (= (sort [(mt/user->id :rasta) (mt/user->id :crowberto)]) + (-> (mt/user-http-request :rasta :get 200 (format "pulse/%d" pulse-id)) + vector + recipient-ids)))) + + (with-redefs [mt.api.u/segmented-user? (constantly true)] + (is (= [(mt/user->id :rasta)] + (-> (mt/user-http-request :rasta :get 200 "pulse/") + recipient-ids))) + + (is (= [(mt/user->id :rasta)] + (-> (mt/user-http-request :rasta :get 200 (format "pulse/%d" pulse-id)) + vector + recipient-ids))))))))) + +(deftest sandboxed-users-cant-delete-pulse-recipients + (testing "When sandboxed users update a pulse, Metabase users in the recipients list are not deleted, even if they + are not included in the request." + (mt/with-temp* [Pulse [{pulse-id :id} {:name "my pulse"}] + PulseChannel [{pc-id :id :as pc} {:pulse_id pulse-id + :channel_type :email + :details {:emails ["asdf@metabase.com"]}}] + PulseChannelRecipient [_ {:pulse_channel_id pc-id, :user_id (mt/user->id :crowberto)}] + PulseChannelRecipient [_ {:pulse_channel_id pc-id, :user_id (mt/user->id :rasta)}]] + + (mt/with-test-user :rasta + (with-redefs [mt.api.u/segmented-user? (constantly true)] + ;; Rasta, a sandboxed user, updates the pulse, but does not include Crowberto in the recipients list + (mt/user-http-request :rasta :put 200 (format "pulse/%d" pulse-id) + {:channels [(assoc pc :recipients [{:id (mt/user->id :rasta)}])]})) + + ;; Check that both Rasta and Crowberto are still recipients + (is (= (sort [(mt/user->id :rasta) (mt/user->id :crowberto)]) + (->> (api.alert/email-channel (pulse/retrieve-pulse pulse-id)) :recipients (map :id) sort))) + + (with-redefs [mt.api.u/segmented-user? (constantly false)] + ;; Rasta, a non-sandboxed user, updates the pulse, but does not include Crowberto in the recipients list + (mt/user-http-request :rasta :put 200 (format "pulse/%d" pulse-id) + {:channels [(assoc pc :recipients [{:id (mt/user->id :rasta)}])]}) + + + ;; Crowberto should now be removed as a recipient + (is (= [(mt/user->id :rasta)] + (->> (api.alert/email-channel (pulse/retrieve-pulse pulse-id)) :recipients (map :id) sort)))))))) diff --git a/enterprise/backend/test/metabase_enterprise/sandbox/query_processor/middleware/row_level_restrictions_test.clj b/enterprise/backend/test/metabase_enterprise/sandbox/query_processor/middleware/row_level_restrictions_test.clj index 1255afc7d344..8baa83b31030 100644 --- a/enterprise/backend/test/metabase_enterprise/sandbox/query_processor/middleware/row_level_restrictions_test.clj +++ b/enterprise/backend/test/metabase_enterprise/sandbox/query_processor/middleware/row_level_restrictions_test.clj @@ -273,6 +273,15 @@ :attributes {"something_random" 50}} (mt/run-mbql-query venues {:aggregation [[:count]]}))))) + (testing (str "When processing a query that requires a user attribute and that user attribute is nil, throw an " + "exception letting the user know it's missing") + (is (thrown-with-msg? + clojure.lang.ExceptionInfo + #"Query requires user attribute `cat`" + (mt/with-gtaps {:gtaps {:venues (venues-category-mbql-gtap-def)} + :attributes {"cat" nil}} + (mt/run-mbql-query venues {:aggregation [[:count]]}))))) + (testing "Another basic test, same as above, but with a numeric string that needs to be coerced" (mt/with-gtaps {:gtaps {:venues (venues-category-mbql-gtap-def)} :attributes {"cat" "50"}} @@ -1007,12 +1016,11 @@ (mt/dataset sample-dataset ;; with-gtaps creates a new copy of the database. So make sure to do that before anything else. Gets really ;; confusing when `(mt/id)` and friends change value halfway through the test - (mt/with-gtaps {:gtaps {:products - {:remappings {:category - ["dimension" - [:field (mt/id :products :category) - nil]]}}} - :attributes {"category" nil}} + (mt/with-gtaps {:gtaps {:products + {:remappings {:category + ["dimension" + [:field (mt/id :products :category) + nil]]}}}} (mt/with-persistence-enabled [persist-models!] (mt/with-temp* [Card [model {:dataset true :dataset_query (mt/mbql-query @@ -1020,8 +1028,9 @@ ;; note does not include the field we have to filter on. No way ;; to use the sandbox filter on the cached table {:fields [$id $price]})}]] - ;; persist model - (persist-models!) + ;; persist model (as admin, so sandboxing is not applied to the persisted query) + (mt/with-test-user :crowberto + (persist-models!)) (let [persisted-info (db/select-one 'PersistedInfo :database_id (mt/id) :card_id (:id model))] diff --git a/enterprise/backend/test/metabase_enterprise/search/scoring_test.clj b/enterprise/backend/test/metabase_enterprise/search/scoring_test.clj index 2b0eaf098dd8..a96cb7c42452 100644 --- a/enterprise/backend/test/metabase_enterprise/search/scoring_test.clj +++ b/enterprise/backend/test/metabase_enterprise/search/scoring_test.clj @@ -1,5 +1,8 @@ (ns metabase-enterprise.search.scoring-test - (:require [clojure.test :refer :all] + (:require [cheshire.core :as json] + [clojure.math.combinatorics :as math.combo] + [clojure.string :as str] + [clojure.test :refer :all] [java-time :as t] [metabase-enterprise.search.scoring :as ee-scoring] [metabase.public-settings.premium-features :as premium-features] @@ -18,67 +21,108 @@ (is (= [1 3 2] (score [(item 1 "verified") (item 2 nil) (item 3 nil)])))))) (defn- ee-score - [search-string] - (fn [item] - (with-redefs [#_{:clj-kondo/ignore [:deprecated-var]} premium-features/enable-enhancements? (constantly true)] - (-> (scoring/score-and-result search-string item) :score)))) + [search-string item] + (with-redefs [#_{:clj-kondo/ignore [:deprecated-var]} + premium-features/enable-enhancements? (constantly true)] + (-> (scoring/score-and-result search-string item) :score))) (defn- oss-score - [search-string] - (fn [item] - (with-redefs [#_{:clj-kondo/ignore [:deprecated-var]} premium-features/enable-enhancements? (constantly false)] - (-> (scoring/score-and-result search-string item) :score)))) + [search-string item] + (with-redefs [#_{:clj-kondo/ignore [:deprecated-var]} + premium-features/enable-enhancements? (constantly false)] + (-> (scoring/score-and-result search-string item) :score))) (deftest official-collection-tests (testing "it should bump up the value of items in official collections" ;; using the ee implementation that isn't wrapped by enable-enhancements? check - (let [search-string "custom expression examples" - ee-score (ee-score search-string) - oss-score (oss-score search-string) - labeled-results {:a {:name "custom expression examples" :model "dashboard"} - :b {:name "examples of custom expressions" :model "dashboard"} - :c {:name "customer success stories" - :dashboardcard_count 50 - :updated_at (t/offset-date-time) - :collection_position 1 - :model "dashboard"} - :d {:name "customer examples of bad sorting" :model "dashboard"}} - {:keys [a b c d]} labeled-results] + (let [search-string "custom expression examples" + a {:id "a" :name "custom expression examples" :model "dashboard"} + b {:id "b" :name "examples of custom expressions" :model "dashboard"} + c {:id "c" + :name "customer success stories" + :dashboardcard_count 50 + :updated_at (t/offset-date-time) + :collection_position 1 + :model "dashboard"} + d {:id "d" :name "customer examples of bad sorting" :model "dashboard"}] (doseq [item [a b c d]] - (is (> (ee-score (assoc item :collection_authority_level "official")) (ee-score item)) - (str "Item not greater for model: " (:model item)))) - (let [items (shuffle [a b c d])] - (is (= (sort-by oss-score items) - ;; assert that the ordering remains the same even if scores are slightly different - (sort-by ee-score items))) - (is (= ["customer examples of bad sorting" - "customer success stories" - "examples of custom expressions" - "custom expression examples"] - (map :name (sort-by oss-score [a b c d])))) - (is (= ["customer success stories" - "customer examples of bad sorting" ;; bumped up slightly in results - "examples of custom expressions" - "custom expression examples"] - (map :name (sort-by ee-score [a b c - (assoc d :collection_authority_level "official")]))))))) + (is (> (ee-score search-string (assoc item :collection_authority_level "official")) + (ee-score search-string item)) + (str "Score should be greater for item: " item " vs " (assoc item :collection_authority_level "official")))) + (is (= ["customer examples of bad sorting" + "customer success stories" + "examples of custom expressions" + "custom expression examples"] + (mapv :name (sort-by #(oss-score search-string %) + (shuffle [a b c d]))))) + (is (= ["customer examples of bad sorting" + "customer success stories" + "examples of custom expressions" + "custom expression examples"] + (mapv :name (sort-by #(ee-score search-string %) + (shuffle [a b c (assoc d :collection_authority_level "official")]))))))) (testing "It should bump up the value of verified items" - (let [search-string "foo" - dashboard-count #(assoc % :dashboardcard_count 0) - ee-score (comp (ee-score search-string) dashboard-count) - oss-score (comp (oss-score search-string) dashboard-count) - labeled-results {:a {:name "foobar" :model "card" :id :a} - :b {:name "foo foo" :model "card" :id :b} - :c {:name "foo foo foo" :model "card" :id :c}} - {:keys [a b c]} labeled-results] + (let [ss "foo" + a {:name "foobar" + :model "card" + :id :a + :dashboardcard_count 0} + b {:name "foo foo" + :model "card" + :id :b + :dashboardcard_count 0} + c {:name "foo foo foo" + :model "card" + :id :c + :dashboardcard_count 0}] (doseq [item [a b c]] - (is (> (ee-score (assoc item :moderated_status "verified")) (ee-score item)) + (is (> (ee-score ss (assoc item :moderated_status "verified")) + (ee-score ss item)) (str "Item not greater for model: " (:model item)))) (let [items (shuffle [a b c])] - (is (= (sort-by oss-score items) (sort-by ee-score items)))) - ;; a is sorted lowest here (sort-by is ascending) - (is (= [:a :c :b] (map :id (sort-by ee-score [a b c])))) - ;; a is verified and is now last or highest score - (is (= [:c :b :a] + (is (= (sort-by #(oss-score ss %) items) + (sort-by #(ee-score ss %) items)))) + (is (= [:c :b :a] (map :id (sort-by #(ee-score ss %) [a b c])))) + ;; c is verified and is now last or highest score + (is (= [:b :a :c] (map :id - (sort-by ee-score [(assoc a :moderated_status "verified") b c]))))))) + (sort-by #(ee-score ss %) + [a + b + (assoc c :moderated_status "verified")]))))))) + +(defn- all-permutations-all-orders + "(all-permutations-all-orders [1]) ;; => [[] [1]] + (all-permutations-all-orders [1 2]) + ;; => [[] [1] [2] [1 2] [2 1]] + (all-permutations-all-orders [1 2 3]) + ;; => [[] ;; size 0 + ;; [1] [2] [3] ;; size 1 + ;; [1 2] [2 1] [1 3] [3 1] [2 3] [3 2] ;; size 2 + ;; [1 2 3] [1 3 2] [2 1 3] [2 3 1] [3 1 2] [3 2 1]] ;; size 3 + " + [values] + {:pre [(> 10 (count values))]} + (mapv vec (mapcat math.combo/permutations (math.combo/subsets values)))) + +(defn test-corups [words] + (let [corpus (->> words + all-permutations-all-orders + (mapv #(str/join " " %)) + (remove #{""})) + the-query (json/generate-string {:type :query :query {:source-table 1}}) + ->query (fn [n] {:name n :dataset_query the-query}) + results (map ->query corpus)] + (doseq [search-string corpus] + (is (= search-string + (-> (scoring/top-results + results + 1 + (map #(metabase.search.scoring/score-and-result search-string %))) + first + :name)))))) + +(deftest identical-results-result-in-identical-hits + (test-corups ["foo" "bar"]) + (test-corups ["foo" "bar" "baz"]) + (test-corups ["foo" "bar" "baz" "quux"])) diff --git a/enterprise/backend/test/metabase_enterprise/serialization/api/serialize_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/api/serialize_test.clj index cd92bca0b2c3..b45ee49e7132 100644 --- a/enterprise/backend/test/metabase_enterprise/serialization/api/serialize_test.clj +++ b/enterprise/backend/test/metabase_enterprise/serialization/api/serialize_test.clj @@ -10,7 +10,9 @@ (defn- do-serialize-data-model [f] (premium-features-test/with-premium-features #{:serialization} - (mt/with-temp* [Collection [{collection-id :id}] + (mt/with-temp* [Collection [{collection-id :id + collection-eid :entity_id + collection-slug :slug}] Dashboard [{dashboard-id :id} {:collection_id collection-id}] Card [{card-id :id} {:collection_id collection-id}] DashboardCard [_ {:card_id card-id, :dashboard_id dashboard-id}]] @@ -19,11 +21,15 @@ (is (= collection-id (db/select-one-field :collection_id Card :id card-id)))) (mt/with-temp-dir [dir "serdes-dir"] - (f {:collection-id collection-id, :dir dir}))))) + (f {:collection-id collection-id + :collection-filename (if collection-slug + (str collection-eid "_" collection-slug) + collection-eid) + :dir dir}))))) (deftest serialize-data-model-happy-path-test (do-serialize-data-model - (fn [{:keys [collection-id dir]}] + (fn [{:keys [collection-id collection-filename dir]}] (is (= {:status "ok"} (mt/user-http-request :crowberto :post 200 "ee/serialization/serialize/data-model" {:collection_ids [collection-id] @@ -35,26 +41,18 @@ (path-files (apply u.files/get-path dir path-components)))] (is (= (map #(.toString (u.files/get-path (System/getProperty "java.io.tmpdir") "serdes-dir" %)) - ["Card" "Collection" "Dashboard" "settings.yaml"]) + ["collections" "settings.yaml"]) (files))) (testing "subdirs" - (testing "Card" + (testing "cards" + (is (= 1 + (count (files "collections" collection-filename "cards"))))) + (testing "collections" (is (= 1 - (count (files "Card"))))) - (testing "Collection" + (count (remove #{"cards" "dashboards" "timelines"} (files "collections")))))) + (testing "dashboards" (is (= 1 - (count (files "Collection"))))) - (testing "Dashboard" - (is (= 2 - (count (files "Dashboard")))) - (let [[f1 f2] (files "Dashboard") - [path-1 path-2] (map u.files/get-path [f1 f2])] - (testing "Should have one subdirectory" - (is (= 1 - (count (filter true? (map u.files/regular-file? [path-1 path-2])))))) - (let [subdirectory-path (first (remove u.files/regular-file? [path-1 path-2]))] - (is (= 1 - (count (path-files subdirectory-path))))))))))))) + (count (files "collections" collection-filename "dashboards"))))))))))) (deftest serialize-data-model-validation-test (do-serialize-data-model diff --git a/enterprise/backend/test/metabase_enterprise/serialization/v2/backfill_ids_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/v2/backfill_ids_test.clj new file mode 100644 index 000000000000..d3ec665b9d3d --- /dev/null +++ b/enterprise/backend/test/metabase_enterprise/serialization/v2/backfill_ids_test.clj @@ -0,0 +1,66 @@ +(ns metabase-enterprise.serialization.v2.backfill-ids-test + (:require + [clojure.test :refer :all] + [metabase-enterprise.serialization.test-util :as ts] + [metabase-enterprise.serialization.v2.backfill-ids :as serdes.backfill] + [metabase.models :refer [Collection]] + [toucan.db :as db])) + +(deftest backfill-needed-test + (ts/with-empty-h2-app-db + (ts/with-temp-dpc [Collection [{c1-id :id} {:name "some collection"}] + Collection [{c2-id :id} {:name "other collection"}] + ;; These two deliberately have the same name! + Collection [{c3-id :id} {:name "child collection" + :location (str "/" c1-id "/")}] + Collection [{c4-id :id} {:name "child collection" + :location (str "/" c2-id "/")}]] + + (let [coll-ids [c1-id c2-id c3-id c4-id] + all-eids #(db/select-field :entity_id Collection :id [:in coll-ids])] + (testing "all collections have entity_ids" + (is (every? some? (all-eids)))) + + (testing "removing the entity_ids" + (doseq [id coll-ids] + (db/update! Collection id :entity_id nil)) + (is (every? nil? (all-eids)))) + + (testing "backfill now recreates them" + (serdes.backfill/backfill-ids-for Collection) + (is (every? some? (all-eids)))))))) + +(deftest no-overwrite-test + (ts/with-empty-h2-app-db + (ts/with-temp-dpc [Collection [{c1-id :id c1-eid :entity_id} {:name "some collection"}] + Collection [{c2-id :id} {:name "other collection"}]] + (testing "deleting the entity_id for one of them" + (db/update! Collection c2-id {:entity_id nil}) + (is (= #{c1-eid nil} + (db/select-field :entity_id Collection)))) + + (testing "backfill" + (serdes.backfill/backfill-ids-for Collection) + (testing "sets a blank entity_id" + (is (some? (db/select-one-field :entity_id Collection :id c2-id)))) + (testing "does not change the original entity_id" + (is (= c1-eid (db/select-one-field :entity_id Collection :id c1-id)))))))) + +(deftest repeatable-test + (ts/with-empty-h2-app-db + (ts/with-temp-dpc [Collection [{c1-eid :entity_id} {:name "some collection"}] + Collection [{c2-id :id} {:name "other collection"}]] + (testing "deleting the entity_id for one of them" + (db/update! Collection c2-id {:entity_id nil}) + (is (= #{c1-eid nil} + (db/select-field :entity_id Collection)))) + + (testing "backfilling twice" + (serdes.backfill/backfill-ids-for Collection) + (let [first-eid (db/select-one-field :entity_id Collection :id c2-id)] + (db/update! Collection c2-id {:entity_id nil}) + (is (= #{c1-eid nil} + (db/select-field :entity_id Collection))) + (serdes.backfill/backfill-ids-for Collection) + (testing "produces the same entity_id both times" + (is (= first-eid (db/select-one-field :entity_id Collection :id c2-id))))))))) diff --git a/enterprise/backend/test/metabase_enterprise/serialization/v2/e2e/yaml_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/v2/e2e/yaml_test.clj index f290aae64eba..5decc7568ad7 100644 --- a/enterprise/backend/test/metabase_enterprise/serialization/v2/e2e/yaml_test.clj +++ b/enterprise/backend/test/metabase_enterprise/serialization/v2/e2e/yaml_test.clj @@ -13,18 +13,39 @@ [toucan.db :as db] [yaml.core :as yaml])) -(defn- dir->file-set [dir] +(defn- dir->contents-set [p dir] (->> dir .listFiles - (filter #(.isFile %)) + (filter p) (map #(.getName %)) set)) +(defn- dir->file-set [dir] + (dir->contents-set #(.isFile %) dir)) + +(defn- dir->dir-set [dir] + (dir->contents-set #(.isDirectory %) dir)) + (defn- subdirs [dir] (->> dir .listFiles (remove #(.isFile %)))) +(defn- collections [dir] + (for [coll-dir (subdirs dir) + :when (->> ["cards" "dashboards" "timelines"] + (map #(io/file coll-dir %)) + (filter #(= % coll-dir)) + empty?)] + coll-dir)) + +(defn- file-set [dir] + (let [base (.toPath dir)] + (set (for [file (file-seq dir) + :when (.isFile file) + :let [rel (.relativize base (.toPath file))]] + (mapv str rel))))) + (defn- random-keyword ([prefix n] (random-keyword prefix n 0)) ([prefix n floor] (keyword (str (name prefix) (+ floor (rand-int n)))))) @@ -63,11 +84,12 @@ (let [extraction (atom nil) entities (atom nil)] (ts/with-source-and-dest-dbs - ;; TODO Generating some nested collections would make these tests more robust. + ;; TODO Generating some nested collections would make these tests more robust, but that's difficult. + ;; There are handwritten tests for storage and ingestion that check out the nesting, at least. (ts/with-source-db (testing "insert" (test-gen/insert! - {:collection [[100 {:refs {:personal_owner_id ::rs/omit}}] + {:collection [[100 {:refs {:personal_owner_id ::rs/omit}}] [10 {:refs {:personal_owner_id ::rs/omit} :spec-gen {:namespace :snippets}}]] :database [[10]] @@ -145,89 +167,88 @@ (update m (-> entity :serdes/meta last :model) (fnil conj []) entity)) {} @extraction)) - (is (= 110 (-> @entities (get "Collection") count))))) - - (testing "storage" - (storage.yaml/store! (seq @extraction) dump-dir) - - (testing "for Collections" - (is (= 110 (count (dir->file-set (io/file dump-dir "Collection")))))) - - (testing "for Databases" - (is (= 10 (count (dir->file-set (io/file dump-dir "Database")))))) - - (testing "for Tables" - (is (= 100 - (reduce + (for [db (get @entities "Database") - :let [tables (dir->file-set (io/file dump-dir "Database" (:name db) "Table"))]] - (count tables)))) - "Tables are scattered, so the directories are harder to count")) - - (testing "for Fields" - (is (= 1000 - (reduce + (for [db (get @entities "Database") - table (subdirs (io/file dump-dir "Database" (:name db) "Table"))] - (->> (io/file table "Field") - dir->file-set - count)))) - "Fields are scattered, so the directories are harder to count")) - - (testing "for cards" - (is (= 100 (count (dir->file-set (io/file dump-dir "Card")))))) - - (testing "for dashboards" - (is (= 100 (count (dir->file-set (io/file dump-dir "Dashboard")))))) - - (testing "for dashboard cards" - (is (= 300 - (reduce + (for [dash (get @entities "Dashboard") - :let [card-dir (io/file dump-dir "Dashboard" (:entity_id dash) "DashboardCard")]] - (if (.exists card-dir) - (count (dir->file-set card-dir)) - 0)))))) - - (testing "for dimensions" - (is (= 40 (count (dir->file-set (io/file dump-dir "Dimension")))))) - - (testing "for metrics" - (is (= 30 (count (dir->file-set (io/file dump-dir "Metric")))))) - - (testing "for segments" - (is (= 30 (count (dir->file-set (io/file dump-dir "Segment")))))) - - (testing "for pulses" - (is (= 30 (count (dir->file-set (io/file dump-dir "Pulse")))))) - - (testing "for pulse cards" - (is (= 120 (reduce + (for [pulse (get @entities "Pulse")] - (->> (io/file dump-dir "Pulse" (:entity_id pulse) "PulseCard") - dir->file-set - count)))))) - - (testing "for pulse channels" - (is (= 30 (reduce + (for [pulse (get @entities "Pulse")] - (->> (io/file dump-dir "Pulse" (:entity_id pulse) "PulseChannel") - dir->file-set - count))))) - (is (= 40 (reduce + (for [{:keys [recipients]} (get @entities "PulseChannel")] - (count recipients)))))) - - (testing "for native query snippets" - (is (= 10 (count (dir->file-set (io/file dump-dir "NativeQuerySnippet")))))) - - (testing "for timelines and events" - (is (= 10 (count (dir->file-set (io/file dump-dir "Timeline"))))) - - (is (= 90 (reduce + (for [timeline (get @entities "Timeline")] - (->> (io/file dump-dir "Timeline" (:entity_id timeline) "TimelineEvent") - dir->file-set - count)))))) - - (testing "for settings" - (is (.exists (io/file dump-dir "settings.yaml"))))) + (is (= 110 (-> @entities (get "Collection") count)))) + + (testing "storage" + (storage.yaml/store! (seq @extraction) dump-dir) + + (testing "for Collections" + (is (= 110 (count (for [f (file-set (io/file dump-dir)) + :when (and (= (first f) "collections") + (let [[a b] (take-last 2 f)] + (= b (str a ".yaml"))))] + f))) + "which all go in collections/, even the snippets ones")) + + (testing "for Databases" + (is (= 10 (count (dir->dir-set (io/file dump-dir "databases")))))) + + (testing "for Tables" + (is (= 100 + (reduce + (for [db (get @entities "Database") + :let [tables (dir->dir-set (io/file dump-dir "databases" (:name db) "tables"))]] + (count tables)))) + "Tables are scattered, so the directories are harder to count")) + + (testing "for Fields" + (is (= 1000 + (reduce + (for [db (get @entities "Database") + table (subdirs (io/file dump-dir "databases" (:name db) "tables"))] + (->> (io/file table "fields") + dir->file-set + count)))) + "Fields are scattered, so the directories are harder to count")) + + (testing "for cards" + (is (= 100 (->> (io/file dump-dir "collections") + collections + (map (comp count dir->file-set #(io/file % "cards"))) + (reduce +))))) + + (testing "for dashboards" + (is (= 100 (->> (io/file dump-dir "collections") + collections + (map (comp count dir->file-set #(io/file % "dashboards"))) + (reduce +))))) + + (testing "for timelines" + (is (= 10 (->> (io/file dump-dir "collections") + collections + (map (comp count dir->file-set #(io/file % "timelines"))) + (reduce +))))) + + (testing "for metrics" + (is (= 30 (reduce + (for [db (dir->dir-set (io/file dump-dir "databases")) + table (dir->dir-set (io/file dump-dir "databases" db "tables")) + :let [metrics-dir (io/file dump-dir "databases" db "tables" table "metrics")] + :when (.exists metrics-dir)] + (count (dir->file-set metrics-dir))))))) + + (testing "for segments" + (is (= 30 (reduce + (for [db (dir->dir-set (io/file dump-dir "databases")) + table (dir->dir-set (io/file dump-dir "databases" db "tables")) + :let [segments-dir (io/file dump-dir "databases" db "tables" table "segments")] + :when (.exists segments-dir)] + (count (dir->file-set segments-dir))))))) + + (testing "for native query snippets" + (is (= 10 (->> (io/file dump-dir "snippets") + collections + (map (comp count dir->file-set)) + (reduce +))))) + + (testing "for settings" + (is (.exists (io/file dump-dir "settings.yaml")))))) (testing "ingest and load" (ts/with-dest-db + (testing "ingested set matches extracted set" + (let [extracted-set (set (map (comp #'ingest.yaml/strip-labels serdes.base/serdes-path) @extraction))] + (is (= (count extracted-set) + (count @extraction))) + (is (= extracted-set + (set (keys (#'ingest.yaml/ingest-all (io/file dump-dir)))))))) + (testing "doing ingestion" (is (serdes.load/load-metabase (ingest.yaml/ingest-yaml dump-dir)) "successful")) @@ -306,32 +327,6 @@ (serdes.base/extract-one "Segment" {}) clean-entity))))) - (testing "for pulses" - (doseq [{:keys [entity_id] :as pulse} (get @entities "Pulse")] - (is (= (clean-entity pulse) - (->> (db/select-one 'Pulse :entity_id entity_id) - (serdes.base/extract-one "Pulse" {}) - clean-entity))))) - - (testing "for pulse cards" - (doseq [{:keys [entity_id] :as card} (get @entities "PulseCard")] - (is (= (clean-entity card) - (->> (db/select-one 'PulseCard :entity_id entity_id) - (serdes.base/extract-one "PulseCard" {}) - clean-entity))))) - - (testing "for pulse channels" - (doseq [{:keys [entity_id] :as channel} (get @entities "PulseChannel")] - ;; The :recipients list is in arbitrary order - turn them into sets for comparison. - (is (= (-> channel - (update :recipients set) - clean-entity) - (let [loaded-channel (->> (db/select-one 'PulseChannel :entity_id entity_id) - (serdes.base/extract-one "PulseChannel" {}))] - (-> loaded-channel - (update :recipients set) - clean-entity)))))) - (testing "for native query snippets" (doseq [{:keys [entity_id] :as snippet} (get @entities "NativeQuerySnippet")] (is (= (clean-entity snippet) @@ -344,13 +339,6 @@ (is (= (clean-entity timeline) (->> (db/select-one 'Timeline :entity_id entity_id) (serdes.base/extract-one "Timeline" {}) - clean-entity)))) - - (doseq [{:keys [timeline_id timestamp] :as event} (get @entities "TimelineEvent")] - (is (= (clean-entity event) - (->> (db/select-one-id 'Timeline :entity_id timeline_id) - (db/select-one 'TimelineEvent :timestamp timestamp :timeline_id) - (serdes.base/extract-one "TimelineEvent" {}) clean-entity))))) (testing "for settings" diff --git a/enterprise/backend/test/metabase_enterprise/serialization/v2/extract_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/v2/extract_test.clj index 651079b4726a..4533be1cb8e2 100644 --- a/enterprise/backend/test/metabase_enterprise/serialization/v2/extract_test.clj +++ b/enterprise/backend/test/metabase_enterprise/serialization/v2/extract_test.clj @@ -1,6 +1,7 @@ (ns metabase-enterprise.serialization.v2.extract-test (:require [clojure.set :as set] [clojure.test :refer :all] + [java-time :as t] [metabase-enterprise.serialization.test-util :as ts] [metabase-enterprise.serialization.v2.extract :as extract] [metabase.models :refer [Card Collection Dashboard DashboardCard Database Dimension Field FieldValues Metric @@ -182,16 +183,14 @@ :collection_id dave-coll-id :creator_id mark-id :parameters []}] - DashboardCard [{dc1-id :id - dc1-eid :entity_id} {:card_id c1-id + DashboardCard [_ {:card_id c1-id :dashboard_id dash-id :parameter_mappings [{:parameter_id "12345678" :card_id c1-id :target [:dimension [:field field-id {:source-field field2-id}]]}]}] - DashboardCard [{dc2-id :id - dc2-eid :entity_id} {:card_id c2-id + DashboardCard [_ {:card_id c2-id :dashboard_id other-dash-id :visualization_settings {:table.pivot_column "SOURCE" @@ -213,7 +212,7 @@ {(str "[\"ref\",[\"field\"," field2-id ",null]]") {:column_title "Locus"}}}}]] (testing "table and database are extracted as [db schema table] triples" (let [ser (serdes.base/extract-one "Card" {} (db/select-one 'Card :id c1-id))] - (is (schema= {:serdes/meta (s/eq [{:model "Card" :id c1-eid}]) + (is (schema= {:serdes/meta (s/eq [{:model "Card" :id c1-eid :label "some_question"}]) :table_id (s/eq ["My Database" nil "Schemaless Table"]) :creator_id (s/eq "mark@direstrai.ts") :collection_id (s/eq coll-eid) @@ -237,7 +236,7 @@ (set (serdes.base/serdes-dependencies ser)))))) (let [ser (serdes.base/extract-one "Card" {} (db/select-one 'Card :id c2-id))] - (is (schema= {:serdes/meta (s/eq [{:model "Card" :id c2-eid}]) + (is (schema= {:serdes/meta (s/eq [{:model "Card" :id c2-eid :label "second_question"}]) :table_id (s/eq ["My Database" "PUBLIC" "Schema'd Table"]) :creator_id (s/eq "mark@direstrai.ts") :collection_id (s/eq coll-eid) @@ -268,7 +267,7 @@ (set (serdes.base/serdes-dependencies ser)))))) (let [ser (serdes.base/extract-one "Card" {} (db/select-one 'Card :id c3-id))] - (is (schema= {:serdes/meta (s/eq [{:model "Card" :id c3-eid}]) + (is (schema= {:serdes/meta (s/eq [{:model "Card" :id c3-eid :label "third_question"}]) :table_id (s/eq ["My Database" "PUBLIC" "Schema'd Table"]) :creator_id (s/eq "mark@direstrai.ts") :collection_id (s/eq coll-eid) @@ -312,36 +311,11 @@ {:model "Schema" :id "PUBLIC"} {:model "Table" :id "Schema'd Table"} {:model "Field" :id "Other Field"}]} - (set (serdes.base/serdes-dependencies ser)))))) - - (let [ser (serdes.base/extract-one "DashboardCard" {} (db/select-one 'DashboardCard :id dc1-id))] - (is (schema= {:serdes/meta (s/eq [{:model "Dashboard" :id dash-eid} - {:model "DashboardCard" :id dc1-eid}]) - :dashboard_id (s/eq dash-eid) - :parameter_mappings (s/eq [{:parameter_id "12345678" - :card_id c1-eid - :target [:dimension [:field ["My Database" nil "Schemaless Table" "Some Field"] - {:source-field ["My Database" "PUBLIC" "Schema'd Table" "Other Field"]}]]}]) - :created_at LocalDateTime - s/Keyword s/Any} - ser)) - (is (not (contains? ser :id))) - - (testing "cards depend on their Dashboard and Card, and any fields in their parameter_mappings" - (is (= #{[{:model "Card" :id c1-eid}] - [{:model "Dashboard" :id dash-eid}] - [{:model "Database" :id "My Database"} - {:model "Table" :id "Schemaless Table"} - {:model "Field" :id "Some Field"}] - [{:model "Database" :id "My Database"} - {:model "Schema" :id "PUBLIC"} - {:model "Table" :id "Schema'd Table"} - {:model "Field" :id "Other Field"}]} (set (serdes.base/serdes-dependencies ser))))))) (testing "Cards can be based on other cards" (let [ser (serdes.base/extract-one "Card" {} (db/select-one 'Card :id c5-id))] - (is (schema= {:serdes/meta (s/eq [{:model "Card" :id c5-eid}]) + (is (schema= {:serdes/meta (s/eq [{:model "Card" :id c5-eid :label "dependent_question"}]) :table_id (s/eq ["My Database" "PUBLIC" "Schema'd Table"]) :creator_id (s/eq "mark@direstrai.ts") :collection_id (s/eq coll-eid) @@ -362,43 +336,45 @@ [{:model "Card" :id c4-eid}]} (set (serdes.base/serdes-dependencies ser))))))) - (testing "Dashcard :visualization_settings are included in their deps" - (let [ser (serdes.base/extract-one "DashboardCard" {} (db/select-one 'DashboardCard :id dc2-id))] - (is (schema= {:serdes/meta (s/eq [{:model "Dashboard" :id other-dash} - {:model "DashboardCard" :id dc2-eid}]) - :dashboard_id (s/eq other-dash) - :visualization_settings (s/eq {:table.pivot_column "SOURCE" - :table.cell_column "sum" - :table.columns - [{:name "SOME_FIELD" - :fieldRef [:field ["My Database" nil "Schemaless Table" "Some Field"] nil] - :enabled true} - {:name "sum" - :fieldRef [:field "sum" {:base-type :type/Float}] - :enabled true} - {:name "count" - :fieldRef [:field "count" {:base-type :type/BigInteger}] - :enabled true} - {:name "Average order total" - :fieldRef [:field "Average order total" {:base-type :type/Float}] - :enabled true}] - :column_settings - {"[\"ref\",[\"field\",[\"My Database\",\"PUBLIC\",\"Schema'd Table\",\"Other Field\"],null]]" {:column_title "Locus"}}}) + (testing "Dashboards include their Dashcards" + (let [ser (serdes.base/extract-one "Dashboard" {} (db/select-one 'Dashboard :id other-dash-id))] + (is (schema= {:serdes/meta (s/eq [{:model "Dashboard" :id other-dash :label "dave_s_dash"}]) + :entity_id (s/eq other-dash) + :ordered_cards + [{:visualization_settings (s/eq {:table.pivot_column "SOURCE" + :table.cell_column "sum" + :table.columns + [{:name "SOME_FIELD" + :fieldRef [:field ["My Database" nil "Schemaless Table" "Some Field"] nil] + :enabled true} + {:name "sum" + :fieldRef [:field "sum" {:base-type :type/Float}] + :enabled true} + {:name "count" + :fieldRef [:field "count" {:base-type :type/BigInteger}] + :enabled true} + {:name "Average order total" + :fieldRef [:field "Average order total" {:base-type :type/Float}] + :enabled true}] + :column_settings + {"[\"ref\",[\"field\",[\"My Database\",\"PUBLIC\",\"Schema'd Table\",\"Other Field\"],null]]" {:column_title "Locus"}}}) + :created_at LocalDateTime + s/Keyword s/Any}] :created_at LocalDateTime s/Keyword s/Any} ser)) (is (not (contains? ser :id))) - (testing "DashboardCard depend on their Dashboard and Card, and any fields in their visualization_settings" + (testing "and depend on all referenced cards, including those in visualization_settings" (is (= #{[{:model "Card" :id c2-eid}] - [{:model "Dashboard" :id other-dash}] [{:model "Database" :id "My Database"} {:model "Table" :id "Schemaless Table"} {:model "Field" :id "Some Field"}] [{:model "Database" :id "My Database"} {:model "Schema" :id "PUBLIC"} {:model "Table" :id "Schema'd Table"} - {:model "Field" :id "Other Field"}]} + {:model "Field" :id "Other Field"}] + [{:model "Collection" :id dave-coll-eid}]} (set (serdes.base/serdes-dependencies ser))))))) (testing "collection filtering based on :user option" @@ -431,23 +407,7 @@ (is (= #{dash-eid other-dash} (->> {:collection-set (extract/collection-set-for-user dave-id)} (serdes.base/extract-all "Dashboard") - (by-model "Dashboard")))))) - - (testing "dashboard cards are filtered based on :user" - (testing "dashboard cards whose dashboards are in unowned collections are always returned" - (is (= #{dc1-eid} - (->> {:collection-set (extract/collection-set-for-user nil)} - (serdes.base/extract-all "DashboardCard") - (by-model "DashboardCard")))) - (is (= #{dc1-eid} - (->> {:collection-set (extract/collection-set-for-user mark-id)} - (serdes.base/extract-all "DashboardCard") - (by-model "DashboardCard"))))) - (testing "dashboard cards whose dashboards are in personal collections are returned for the :user" - (is (= #{dc1-eid dc2-eid} - (->> {:collection-set (extract/collection-set-for-user dave-id)} - (serdes.base/extract-all "DashboardCard") - (by-model "DashboardCard"))))))))) + (by-model "Dashboard"))))))))) (deftest dimensions-test (ts/with-empty-h2-app-db @@ -455,60 +415,90 @@ Database [{db-id :id} {:name "My Database"}] Table [{no-schema-id :id} {:name "Schemaless Table" :db_id db-id}] Field [{email-id :id} {:name "email" :table_id no-schema-id}] - Dimension [{dim1-id :id - dim1-eid :entity_id} {:name "Vanilla Dimension" + Dimension [{dim1-eid :entity_id} {:name "Vanilla Dimension" + :field_id email-id + :type "internal" + :created_at (t/minus (t/offset-date-time) + (t/days 3))}] + Dimension [{dim2-eid :entity_id} {:name "Bonus Dimension" :field_id email-id - :type "internal"}] - - ;; Advanced case: :field_id is the foreign key, :human_readable_field_id the real target field. - Table [{this-table :id} {:name "Schema'd Table" + :type "internal" + :created_at (t/minus (t/offset-date-time) + (t/days 6))}] + + ;; Advanced case: Dimension capturing a foreign relationship. + ;; The parent field (Orders.customer_id) is the foreign key. + ;; Dimension.field_id (Customers.id) is the foreign ID field; + ;; Dimension.human_readable_field_id (Customers.name) is what we want to render. + Table [{customers :id} {:name "Customers" :db_id db-id :schema "PUBLIC"}] - Field [{fk-id :id} {:name "foreign_id" :table_id this-table}] - Table [{other-table :id} {:name "Foreign Table" + Field [{cust-id :id} {:name "id" :table_id customers}] + Field [{cust-name :id} {:name "name" :table_id customers}] + Table [{orders :id} {:name "Orders" :db_id db-id :schema "PUBLIC"}] - Field [{target-id :id} {:name "real_field" :table_id other-table}] - Dimension [{dim2-id :id - dim2-eid :entity_id} {:name "Foreign Dimension" + Field [{fk-id :id} {:name "customer_id" + :table_id orders + :fk_target_field_id cust-id}] + Dimension [_ {:name "Customer Name" :type "external" :field_id fk-id - :human_readable_field_id target-id}]] - (testing "vanilla user-created dimensions" - (let [ser (serdes.base/extract-one "Dimension" {} (db/select-one 'Dimension :id dim1-id))] - (is (schema= {:serdes/meta (s/eq [{:model "Dimension" :id dim1-eid}]) - :field_id (s/eq ["My Database" nil "Schemaless Table" "email"]) - :human_readable_field_id (s/eq nil) - :created_at LocalDateTime - s/Keyword s/Any} + :human_readable_field_id cust-name}]] + (testing "dimensions without foreign keys are inlined into their Fields" + (let [ser (serdes.base/extract-one "Field" {} (db/select-one Field :id email-id))] + (is (schema= {:serdes/meta (s/eq [{:model "Database" :id "My Database"} + {:model "Table" :id "Schemaless Table"} + {:model "Field" :id "email"}]) + :dimensions [{(s/optional-key :human_readable_field_id) [(s/maybe s/Str)] + :created_at LocalDateTime + s/Keyword s/Any}] + s/Keyword s/Any} ser)) (is (not (contains? ser :id))) - (testing "depend on the one Field" + (testing "ordered by ascending :created_at" + (is (= [dim2-eid dim1-eid] + (->> ser :dimensions (map :entity_id))))) + + (testing "which depend on just the table" (is (= #{[{:model "Database" :id "My Database"} - {:model "Table" :id "Schemaless Table"} - {:model "Field" :id "email"}]} + {:model "Table" :id "Schemaless Table"}]} (set (serdes.base/serdes-dependencies ser))))))) - (testing "foreign key dimensions" - (let [ser (serdes.base/extract-one "Dimension" {} (db/select-one 'Dimension :id dim2-id))] - (is (schema= {:serdes/meta (s/eq [{:model "Dimension" :id dim2-eid}]) - :field_id (s/eq ["My Database" "PUBLIC" "Schema'd Table" "foreign_id"]) - :human_readable_field_id (s/eq ["My Database" "PUBLIC" "Foreign Table" "real_field"]) - :created_at LocalDateTime - s/Keyword s/Any} + (testing "foreign key dimensions are inlined into their Fields" + (let [ser (serdes.base/extract-one "Field" {} (db/select-one Field :id fk-id))] + (is (schema= {:serdes/meta (s/eq [{:model "Database" :id "My Database"} + {:model "Schema" :id "PUBLIC"} + {:model "Table" :id "Orders"} + {:model "Field" :id "customer_id"}]) + :name (s/eq "customer_id") + :fk_target_field_id (s/eq ["My Database" "PUBLIC" "Customers" "id"]) + :dimensions [{:human_readable_field_id [s/Str] + :created_at LocalDateTime + s/Keyword s/Any}] + s/Keyword s/Any} ser)) (is (not (contains? ser :id))) - (testing "depend on both Fields" + (testing "dimensions are properly inlined" + (is (schema= [{:human_readable_field_id (s/eq ["My Database" "PUBLIC" "Customers" "name"]) + :created_at LocalDateTime + s/Keyword s/Any}] + (:dimensions ser)))) + + (testing "which depend on the Table and both real and human-readable foreign Fields" (is (= #{[{:model "Database" :id "My Database"} {:model "Schema" :id "PUBLIC"} - {:model "Table" :id "Schema'd Table"} - {:model "Field" :id "foreign_id"}] + {:model "Table" :id "Orders"}] + [{:model "Database" :id "My Database"} + {:model "Schema" :id "PUBLIC"} + {:model "Table" :id "Customers"} + {:model "Field" :id "id"}] [{:model "Database" :id "My Database"} {:model "Schema" :id "PUBLIC"} - {:model "Table" :id "Foreign Table"} - {:model "Field" :id "real_field"}]} + {:model "Table" :id "Customers"} + {:model "Field" :id "name"}]} (set (serdes.base/serdes-dependencies ser)))))))))) (deftest metrics-test @@ -528,7 +518,7 @@ :aggregation [[:sum [:field field-id nil]]]}}]] (testing "metrics" (let [ser (serdes.base/extract-one "Metric" {} (db/select-one 'Metric :id m1-id))] - (is (schema= {:serdes/meta (s/eq [{:model "Metric" :id m1-eid :label "My Metric"}]) + (is (schema= {:serdes/meta (s/eq [{:model "Metric" :id m1-eid :label "my_metric"}]) :table_id (s/eq ["My Database" nil "Schemaless Table"]) :creator_id (s/eq "ann@heart.band") :definition (s/eq {:source-table ["My Database" nil "Schemaless Table"] @@ -570,7 +560,7 @@ (let [ser (serdes.base/extract-one "NativeQuerySnippet" {} (db/select-one 'NativeQuerySnippet :id s1-id))] (is (schema= {:serdes/meta (s/eq [{:model "NativeQuerySnippet" :id s1-eid - :label "Snippet 1"}]) + :label "snippet_1"}]) :collection_id (s/eq coll-eid) :creator_id (s/eq "ann@heart.band") :created_at OffsetDateTime @@ -586,7 +576,7 @@ (let [ser (serdes.base/extract-one "NativeQuerySnippet" {} (db/select-one 'NativeQuerySnippet :id s2-id))] (is (schema= {:serdes/meta (s/eq [{:model "NativeQuerySnippet" :id s2-eid - :label "Snippet 2"}]) + :label "snippet_2"}]) (s/optional-key :collection_id) (s/eq nil) :creator_id (s/eq "ann@heart.band") :created_at OffsetDateTime @@ -613,14 +603,14 @@ line-eid :entity_id} {:name "Populated Timeline" :collection_id coll-id :creator_id ann-id}] - TimelineEvent [{e1-id :id} {:name "First Event" + TimelineEvent [_ {:name "First Event" :creator_id ann-id :timestamp #t "2020-04-11T00:00Z" :timeline_id line-id}]] (testing "timelines" (testing "with no events" (let [ser (serdes.base/extract-one "Timeline" {} (db/select-one 'Timeline :id empty-id))] - (is (schema= {:serdes/meta (s/eq [{:model "Timeline" :id empty-eid}]) + (is (schema= {:serdes/meta (s/eq [{:model "Timeline" :id empty-eid :label "empty_timeline"}]) :collection_id (s/eq coll-eid) :creator_id (s/eq "ann@heart.band") :created_at OffsetDateTime @@ -633,37 +623,24 @@ (set (serdes.base/serdes-dependencies ser))))))) (testing "with events" - (let [ser (serdes.base/extract-one "Timeline" {} (db/select-one 'Timeline :id line-id))] - (is (schema= {:serdes/meta (s/eq [{:model "Timeline" :id line-eid}]) + (let [ser (serdes.base/extract-one "Timeline" {} (db/select-one 'Timeline :id line-id)) + stamp "2020-04-11T00:00:00Z"] + (is (schema= {:serdes/meta (s/eq [{:model "Timeline" :id line-eid :label "populated_timeline"}]) :collection_id (s/eq coll-eid) :creator_id (s/eq "ann@heart.band") :created_at OffsetDateTime + :events [{:timestamp (s/eq stamp) + :creator_id (s/eq "ann@heart.band") + :created_at OffsetDateTime + s/Keyword s/Any}] s/Keyword s/Any} ser)) (is (not (contains? ser :id))) + (is (not (contains? (-> ser :events first) :id))) (testing "depend on the Collection" (is (= #{[{:model "Collection" :id coll-eid}]} - (set (serdes.base/serdes-dependencies ser)))))))) - - (testing "timeline events" - (let [ser (serdes.base/extract-one "TimelineEvent" {} (db/select-one 'TimelineEvent :id e1-id)) - stamp "2020-04-11T00:00:00Z"] - (is (schema= {:serdes/meta (s/eq [{:model "Timeline" :id line-eid} - {:model "TimelineEvent" - :id stamp - :label "First Event"}]) - :timestamp (s/eq stamp) - :timeline_id (s/eq line-eid) - :creator_id (s/eq "ann@heart.band") - :created_at OffsetDateTime - s/Keyword s/Any} - ser)) - (is (not (contains? ser :id))) - - (testing "depend on the Timeline" - (is (= #{[{:model "Timeline" :id line-eid}]} - (set (serdes.base/serdes-dependencies ser)))))))))) + (set (serdes.base/serdes-dependencies ser))))))))))) (deftest segments-test (ts/with-empty-h2-app-db @@ -682,7 +659,7 @@ :filter [:< [:field field-id nil] 18]}}]] (testing "segment" (let [ser (serdes.base/extract-one "Segment" {} (db/select-one 'Segment :id s1-id))] - (is (schema= {:serdes/meta (s/eq [{:model "Segment" :id s1-eid :label "My Segment"}]) + (is (schema= {:serdes/meta (s/eq [{:model "Segment" :id s1-eid :label "my_segment"}]) :table_id (s/eq ["My Database" nil "Schemaless Table"]) :creator_id (s/eq "ann@heart.band") :definition (s/eq {:source-table ["My Database" nil "Schemaless Table"] @@ -772,7 +749,9 @@ :dashboard_id dash-id}]] (testing "pulse with neither collection nor dashboard" (let [ser (serdes.base/extract-one "Pulse" {} (db/select-one 'Pulse :id p-none-id))] - (is (schema= {:serdes/meta (s/eq [{:model "Pulse" :id p-none-eid}]) + (is (schema= {:serdes/meta (s/eq [{:model "Pulse" + :id p-none-eid + :label "pulse_w_o_collection_or_dashboard"}]) :creator_id (s/eq "ann@heart.band") (s/optional-key :dashboard_id) (s/eq nil) (s/optional-key :collection_id) (s/eq nil) @@ -787,7 +766,9 @@ (testing "pulse with just collection" (let [ser (serdes.base/extract-one "Pulse" {} (db/select-one 'Pulse :id p-coll-id))] - (is (schema= {:serdes/meta (s/eq [{:model "Pulse" :id p-coll-eid}]) + (is (schema= {:serdes/meta (s/eq [{:model "Pulse" + :id p-coll-eid + :label "pulse_with_only_collection"}]) :creator_id (s/eq "ann@heart.band") (s/optional-key :dashboard_id) (s/eq nil) :collection_id (s/eq coll-eid) @@ -802,7 +783,9 @@ (testing "pulse with just dashboard" (let [ser (serdes.base/extract-one "Pulse" {} (db/select-one 'Pulse :id p-dash-id))] - (is (schema= {:serdes/meta (s/eq [{:model "Pulse" :id p-dash-eid}]) + (is (schema= {:serdes/meta (s/eq [{:model "Pulse" + :id p-dash-eid + :label "pulse_with_only_dashboard"}]) :creator_id (s/eq "ann@heart.band") :dashboard_id (s/eq dash-eid) (s/optional-key :collection_id) (s/eq nil) @@ -817,7 +800,9 @@ (testing "pulse with both collection and dashboard" (let [ser (serdes.base/extract-one "Pulse" {} (db/select-one 'Pulse :id p-both-id))] - (is (schema= {:serdes/meta (s/eq [{:model "Pulse" :id p-both-eid}]) + (is (schema= {:serdes/meta (s/eq [{:model "Pulse" + :id p-both-eid + :label "pulse_with_both_collection_and_dashboard"}]) :creator_id (s/eq "ann@heart.band") :dashboard_id (s/eq dash-eid) :collection_id (s/eq coll-eid) @@ -908,10 +893,10 @@ ser)) (is (not (contains? ser :id))) - (testing "depends on the pulse, card and dashcard" + (testing "depends on the pulse, card and parent dashboard" (is (= #{[{:model "Pulse" :id sub-eid}] [{:model "Card" :id card1-eid}] - [{:model "Dashboard" :id dash-eid} {:model "DashboardCard" :id dashcard-eid}]} + [{:model "Dashboard" :id dash-eid}]} (set (serdes.base/serdes-dependencies ser)))))))))) (deftest selective-serialization-basic-test @@ -960,9 +945,9 @@ :collection_id coll1-id :creator_id mark-id}] - DashboardCard [{dc1-1-eid :entity_id} {:card_id c1-1-id + DashboardCard [_ {:card_id c1-1-id :dashboard_id dash1-id}] - DashboardCard [{dc1-2-eid :entity_id} {:card_id c1-2-id + DashboardCard [_ {:card_id c1-2-id :dashboard_id dash1-id}] ;; Second dashboard, in the middle collection. @@ -988,9 +973,9 @@ :collection_id coll2-id :creator_id mark-id}] - DashboardCard [{dc2-1-eid :entity_id} {:card_id c2-1-id + DashboardCard [_ {:card_id c2-1-id :dashboard_id dash2-id}] - DashboardCard [{dc2-2-eid :entity_id} {:card_id c2-2-id + DashboardCard [_ {:card_id c2-2-id :dashboard_id dash2-id}] ;; Third dashboard, in the grandchild collection. @@ -1016,76 +1001,52 @@ :collection_id coll3-id :creator_id mark-id}] - DashboardCard [{dc3-1-eid :entity_id} {:card_id c3-1-id + DashboardCard [_ {:card_id c3-1-id :dashboard_id dash3-id}] - DashboardCard [{dc3-2-eid :entity_id} {:card_id c3-2-id + DashboardCard [_ {:card_id c3-2-id :dashboard_id dash3-id}]] - (testing "selecting a dashboard gets its dashcards and cards as well" + (testing "selecting a dashboard gets all cards its dashcards depend on" (testing "grandparent dashboard" - (is (= #{[{:model "Dashboard" :id dash1-eid}] - [{:model "Dashboard" :id dash1-eid} - {:model "DashboardCard" :id dc1-1-eid}] - [{:model "Dashboard" :id dash1-eid} - {:model "DashboardCard" :id dc1-2-eid}] - [{:model "Card" :id c1-1-eid}] - [{:model "Card" :id c1-2-eid}]} + (is (= #{[{:model "Dashboard" :id dash1-eid :label "dashboard_1"}] + [{:model "Card" :id c1-1-eid :label "question_1_1"}] + [{:model "Card" :id c1-2-eid :label "question_1_2"}]} (->> (extract/extract-subtrees {:targets [["Dashboard" dash1-id]]}) (map serdes.base/serdes-path) set)))) (testing "middle dashboard" - (is (= #{[{:model "Dashboard" :id dash2-eid}] - [{:model "Dashboard" :id dash2-eid} - {:model "DashboardCard" :id dc2-1-eid}] - [{:model "Dashboard" :id dash2-eid} - {:model "DashboardCard" :id dc2-2-eid}] - [{:model "Card" :id c2-1-eid}] - [{:model "Card" :id c2-2-eid}]} + (is (= #{[{:model "Dashboard" :id dash2-eid :label "dashboard_2"}] + [{:model "Card" :id c2-1-eid :label "question_2_1"}] + [{:model "Card" :id c2-2-eid :label "question_2_2"}]} (->> (extract/extract-subtrees {:targets [["Dashboard" dash2-id]]}) (map serdes.base/serdes-path) set)))) (testing "grandchild dashboard" - (is (= #{[{:model "Dashboard" :id dash3-eid}] - [{:model "Dashboard" :id dash3-eid} - {:model "DashboardCard" :id dc3-1-eid}] - [{:model "Dashboard" :id dash3-eid} - {:model "DashboardCard" :id dc3-2-eid}] - [{:model "Card" :id c3-1-eid}] - [{:model "Card" :id c3-2-eid}]} + (is (= #{[{:model "Dashboard" :id dash3-eid :label "dashboard_3"}] + [{:model "Card" :id c3-1-eid :label "question_3_1"}] + [{:model "Card" :id c3-2-eid :label "question_3_2"}]} (->> (extract/extract-subtrees {:targets [["Dashboard" dash3-id]]}) (map serdes.base/serdes-path) set))))) (testing "selecting a collection gets all its contents" (let [grandchild-paths #{[{:model "Collection" :id coll3-eid :label "grandchild_collection"}] - [{:model "Dashboard" :id dash3-eid}] - [{:model "Dashboard" :id dash3-eid} - {:model "DashboardCard" :id dc3-1-eid}] - [{:model "Dashboard" :id dash3-eid} - {:model "DashboardCard" :id dc3-2-eid}] - [{:model "Card" :id c3-1-eid}] - [{:model "Card" :id c3-2-eid}] - [{:model "Card" :id c3-3-eid}]} + [{:model "Dashboard" :id dash3-eid :label "dashboard_3"}] + [{:model "Card" :id c3-1-eid :label "question_3_1"}] + [{:model "Card" :id c3-2-eid :label "question_3_2"}] + [{:model "Card" :id c3-3-eid :label "question_3_3"}]} middle-paths #{[{:model "Collection" :id coll2-eid :label "nested_collection"}] - [{:model "Dashboard" :id dash2-eid}] - [{:model "Dashboard" :id dash2-eid} - {:model "DashboardCard" :id dc2-1-eid}] - [{:model "Dashboard" :id dash2-eid} - {:model "DashboardCard" :id dc2-2-eid}] - [{:model "Card" :id c2-1-eid}] - [{:model "Card" :id c2-2-eid}] - [{:model "Card" :id c2-3-eid}]} + [{:model "Dashboard" :id dash2-eid :label "dashboard_2"}] + [{:model "Card" :id c2-1-eid :label "question_2_1"}] + [{:model "Card" :id c2-2-eid :label "question_2_2"}] + [{:model "Card" :id c2-3-eid :label "question_2_3"}]} grandparent-paths #{[{:model "Collection" :id coll1-eid :label "some_collection"}] - [{:model "Dashboard" :id dash1-eid}] - [{:model "Dashboard" :id dash1-eid} - {:model "DashboardCard" :id dc1-1-eid}] - [{:model "Dashboard" :id dash1-eid} - {:model "DashboardCard" :id dc1-2-eid}] - [{:model "Card" :id c1-1-eid}] - [{:model "Card" :id c1-2-eid}] - [{:model "Card" :id c1-3-eid}]}] + [{:model "Dashboard" :id dash1-eid :label "dashboard_1"}] + [{:model "Card" :id c1-1-eid :label "question_1_1"}] + [{:model "Card" :id c1-2-eid :label "question_1_2"}] + [{:model "Card" :id c1-3-eid :label "question_1_3"}]}] (testing "grandchild collection has all its own contents" (is (= grandchild-paths ; Includes the third card not found in the collection (->> (extract/extract-subtrees {:targets [["Collection" coll3-id]]}) @@ -1101,3 +1062,22 @@ (->> (extract/extract-subtrees {:targets [["Collection" coll1-id]]}) (map serdes.base/serdes-path) set))))))))) + +(deftest foreign-key-field-test + (ts/with-empty-h2-app-db + (ts/with-temp-dpc [Database [{db-id :id} {:name "My Database"}] + Table [{no-schema-id :id} {:name "Schemaless Table" :db_id db-id}] + Field [{some-field-id :id} {:name "Some Field" :table_id no-schema-id}] + Table [{schema-id :id} {:name "Schema'd Table" + :db_id db-id + :schema "PUBLIC"}] + Field [_ {:name "Other Field" :table_id schema-id}] + Field [{fk-id :id} {:name "Foreign Key" + :table_id schema-id + :fk_target_field_id some-field-id}]] + + (testing "fields that reference foreign keys are properly exported as Field references" + (is (= ["My Database" nil "Schemaless Table" "Some Field"] + (->> (db/select-one Field :id fk-id) + (serdes.base/extract-one "Field" {}) + :fk_target_field_id))))))) diff --git a/enterprise/backend/test/metabase_enterprise/serialization/v2/ingest/yaml_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/v2/ingest/yaml_test.clj index e72c04a76ff2..02f0e40c7a42 100644 --- a/enterprise/backend/test/metabase_enterprise/serialization/v2/ingest/yaml_test.clj +++ b/enterprise/backend/test/metabase_enterprise/serialization/v2/ingest/yaml_test.clj @@ -8,28 +8,32 @@ (deftest basic-ingest-test (ts/with-random-dump-dir [dump-dir "serdesv2-"] - (io/make-parents dump-dir "Collection" "fake") ; Prepare the right directories. + (io/make-parents dump-dir "collections" "1234567890abcdefABCDE_the_label" "fake") ; Prepare the right directories. + (io/make-parents dump-dir "collections" "0987654321zyxwvuABCDE" "fake") + (spit (io/file dump-dir "settings.yaml") (yaml/generate-string {:some-key "with string value" :another-key 7 :blank-key nil})) - (spit (io/file dump-dir "Collection" "fake-id+the_label.yaml") - (yaml/generate-string {:some "made up" :data "here" :entity_id "fake-id" :slug "the_label"})) - (spit (io/file dump-dir "Collection" "no-label.yaml") - (yaml/generate-string {:some "other" :data "in this one" :entity_id "no-label"})) + (spit (io/file dump-dir "collections" "1234567890abcdefABCDE_the_label" "1234567890abcdefABCDE_the_label.yaml") + (yaml/generate-string {:some "made up" :data "here" :entity_id "1234567890abcdefABCDE" :slug "the_label"})) + (spit (io/file dump-dir "collections" "0987654321zyxwvuABCDE" "0987654321zyxwvuABCDE.yaml") + (yaml/generate-string {:some "other" :data "in this one" :entity_id "0987654321zyxwvuABCDE"})) (let [ingestable (ingest.yaml/ingest-yaml dump-dir) - exp-files {[{:model "Collection" :id "fake-id" :label "the_label"}] {:some "made up" - :data "here" - :entity_id "fake-id" - :slug "the_label"} - [{:model "Collection" :id "no-label"}] {:some "other" - :data "in this one" - :entity_id "no-label"} - [{:model "Setting" :id "some-key"}] {:key :some-key :value "with string value"} - [{:model "Setting" :id "another-key"}] {:key :another-key :value 7} - [{:model "Setting" :id "blank-key"}] {:key :blank-key :value nil}}] - (testing "the right set of file is returned by ingest-list" + exp-files {[{:model "Collection" + :id "1234567890abcdefABCDE" + :label "the_label"}] {:some "made up" + :data "here" + :entity_id "1234567890abcdefABCDE" + :slug "the_label"} + [{:model "Collection" :id "0987654321zyxwvuABCDE"}] {:some "other" + :data "in this one" + :entity_id "0987654321zyxwvuABCDE"} + [{:model "Setting" :id "some-key"}] {:key :some-key :value "with string value"} + [{:model "Setting" :id "another-key"}] {:key :another-key :value 7} + [{:model "Setting" :id "blank-key"}] {:key :blank-key :value nil}}] + (testing "the right set of files is returned by ingest-list" (is (= (set (keys exp-files)) (into #{} (ingest/ingest-list ingestable))))) @@ -46,29 +50,30 @@ (deftest flexible-file-matching-test (ts/with-random-dump-dir [dump-dir "serdesv2-"] - (io/make-parents dump-dir "Collection" "fake") - (spit (io/file dump-dir "Collection" "entity-id+human-readable-things.yaml") + (io/make-parents dump-dir "collections" "1234567890abcdefABCDE_human-readable-things" "fake") + (spit (io/file dump-dir "collections" "1234567890abcdefABCDE_human-readable-things" + "1234567890abcdefABCDE_human-readable-things.yaml") (yaml/generate-string {:some "made up" :data "here"})) (let [ingestable (ingest.yaml/ingest-yaml dump-dir) exp {:some "made up" :data "here" - :serdes/meta [{:model "Collection" :id "entity-id"}]}] + :serdes/meta [{:model "Collection" :id "1234567890abcdefABCDE"}]}] (testing "the returned set of files has the human-readable labels" - (is (= #{[{:model "Collection" :id "entity-id" :label "human-readable-things"}]} + (is (= #{[{:model "Collection" :id "1234567890abcdefABCDE" :label "human-readable-things"}]} (into #{} (ingest/ingest-list ingestable))))) (testing "fetching the file with the label works" (is (= exp - (ingest/ingest-one ingestable [{:model "Collection" :id "entity-id" :label "human-readable-things"}])))) + (ingest/ingest-one ingestable [{:model "Collection" :id "1234567890abcdefABCDE" :label "human-readable-things"}])))) (testing "fetching the file without the label also works" (is (= exp - (ingest/ingest-one ingestable [{:model "Collection" :id "entity-id"}]))))))) + (ingest/ingest-one ingestable [{:model "Collection" :id "1234567890abcdefABCDE"}]))))))) (deftest file-name-escaping-test (ts/with-random-dump-dir [dump-dir "serdesv2-"] - (io/make-parents dump-dir "Database" "fake") - (spit (io/file dump-dir "Database" "my data__SLASH__speculations.yaml") + (io/make-parents dump-dir "databases" "my data__SLASH__speculations" "fake") + (spit (io/file dump-dir "databases" "my data__SLASH__speculations" "my data__SLASH__speculations.yaml") (yaml/generate-string {:some "made up" :data "here"})) (let [ingestable (ingest.yaml/ingest-yaml dump-dir) @@ -85,14 +90,14 @@ (deftest keyword-reconstruction-test (ts/with-random-dump-dir [dump-dir "serdesv2-"] - (io/make-parents dump-dir "Card" "fake") - (spit (io/file dump-dir "Card" "some-card.yaml") + (io/make-parents dump-dir "collections" "cards" "fake") + (spit (io/file dump-dir "collections" "cards" "1234567890abcdefABCDE_some_card.yaml") (yaml/generate-string {:visualization_settings {:column_settings {"[\"name\",\"sum\"]" {:number_style "currency"}}}})) (let [ingestable (ingest.yaml/ingest-yaml dump-dir) exp {:visualization_settings {:column_settings {"[\"name\",\"sum\"]" {:number_style "currency"}}} - :serdes/meta [{:model "Card" :id "some-card"}]}] + :serdes/meta [{:model "Card" :id "1234567890abcdefABCDE"}]}] (testing "the file as read in correctly reconstructs keywords only where legal" (is (= exp - (ingest/ingest-one ingestable [{:model "Card" :id "some-card"}]))))))) + (ingest/ingest-one ingestable [{:model "Card" :id "1234567890abcdefABCDE"}]))))))) diff --git a/enterprise/backend/test/metabase_enterprise/serialization/v2/load_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/v2/load_test.clj index a433e6d257db..0480f54303e1 100644 --- a/enterprise/backend/test/metabase_enterprise/serialization/v2/load_test.clj +++ b/enterprise/backend/test/metabase_enterprise/serialization/v2/load_test.clj @@ -1,14 +1,17 @@ (ns metabase-enterprise.serialization.v2.load-test (:require [clojure.test :refer :all] + [java-time :as t] [metabase-enterprise.serialization.test-util :as ts] [metabase-enterprise.serialization.v2.extract :as serdes.extract] [metabase-enterprise.serialization.v2.ingest :as serdes.ingest] [metabase-enterprise.serialization.v2.load :as serdes.load] - [metabase.models :refer [Card Collection Dashboard DashboardCard Database Field FieldValues Metric Pulse - PulseChannel PulseChannelRecipient Segment Table User]] + [metabase.models :refer [Card Collection Dashboard DashboardCard Database Field FieldValues Metric + NativeQuerySnippet Segment Table Timeline TimelineEvent User]] [metabase.models.serialization.base :as serdes.base] - [metabase.models.serialization.hash :as serdes.hash] - [toucan.db :as db])) + [metabase.util :as u] + [schema.core :as s] + [toucan.db :as db]) + (:import java.time.OffsetDateTime)) (defn- no-labels [path] (mapv #(dissoc % :label) path)) @@ -106,43 +109,6 @@ (is (= (format "/%d/%d/" (:id parent-dest) (:id child-dest)) (:location grandchild-dest)))))))))) -(deftest deserialization-upsert-and-dupe-test - (testing "basic collections with their names changing, one without entity_id:" - (let [serialized (atom nil) - c1a (atom nil) - c2a (atom nil) - c1b (atom nil) - c2b (atom nil)] - (ts/with-source-and-dest-dbs - (testing "serializing the two collections" - (ts/with-source-db - (reset! c1b (ts/create! Collection :name "Renamed Collection 1")) - (reset! c2b (ts/create! Collection :name "Collection 2 version 2")) - (db/update! Collection (:id @c2b) {:entity_id nil}) - (reset! c2b (db/select-one Collection :id (:id @c2b))) - (is (nil? (:entity_id @c2b))) - (reset! serialized (into [] (serdes.extract/extract-metabase {}))))) - - (testing "serialization should use identity hashes where no entity_id is defined" - (is (= #{(:entity_id @c1b) - (serdes.hash/identity-hash @c2b)} - (ids-by-model @serialized "Collection")))) - - (testing "deserializing, the name change causes a duplicated collection" - (ts/with-dest-db - (reset! c1a (ts/create! Collection :name "Collection 1" :entity_id (:entity_id @c1b))) - (reset! c2a (ts/create! Collection :name "Collection 2 version 1")) - (db/update! Collection (:id @c2a) {:entity_id nil}) - (reset! c2a (db/select-one Collection :id (:id @c2a))) - (is (nil? (:entity_id @c2b))) - - (serdes.load/load-metabase (ingestion-in-memory @serialized)) - (is (= 3 (db/count Collection)) "Collection 2 versions get duplicated, since the identity-hash changed") - (is (= #{"Renamed Collection 1" - "Collection 2 version 1" - "Collection 2 version 2"} - (set (db/select-field :name Collection)))))))))) - (deftest deserialization-database-table-field-test (testing "databases, tables and fields are nested in namespaces" (let [serialized (atom nil) @@ -151,7 +117,9 @@ db2s (atom nil) db2d (atom nil) t1s (atom nil) - t2s (atom nil)] + t2s (atom nil) + f1s (atom nil) + f2s (atom nil)] (ts/with-source-and-dest-dbs (testing "serializing the two databases" (ts/with-source-db @@ -159,6 +127,8 @@ (reset! t1s (ts/create! Table :name "posts" :db_id (:id @db1s))) (reset! db2s (ts/create! Database :name "db2")) (reset! t2s (ts/create! Table :name "posts" :db_id (:id @db2s))) ; Deliberately the same name! + (reset! f1s (ts/create! Field :name "Target Field" :table_id (:id @t1s))) + (reset! f2s (ts/create! Field :name "Foreign Key" :table_id (:id @t2s) :fk_target_field_id (:id @f1s))) (reset! serialized (into [] (serdes.extract/extract-metabase {}))))) (testing "serialization of databases is based on the :name" @@ -172,6 +142,14 @@ (map :db_id) set)))) + (testing "foreign key references are serialized as a field path" + (is (= ["db1" nil "posts" "Target Field"] + (->> @serialized + (filter #(-> % :serdes/meta last :model (= "Field"))) + (filter #(-> % :table_id (= ["db2" nil "posts"]))) + (keep :fk_target_field_id) + first)))) + (testing "deserialization works properly, keeping the same-named tables apart" (ts/with-dest-db (serdes.load/load-metabase (ingestion-in-memory @serialized)) @@ -186,76 +164,6 @@ (is (db/exists? Table :name "posts" :db_id (:id @db1d))) (is (db/exists? Table :name "posts" :db_id (:id @db2d))))))))) -(deftest pulse-channel-recipient-merging-test - (testing "pulse channel recipients are listed as emails on a channel, then merged with the existing ones" - (let [serialized (atom nil) - u1s (atom nil) - u2s (atom nil) - u3s (atom nil) - pulse-s (atom nil) - pc1s (atom nil) - pc2s (atom nil) - pcr1s (atom nil) - pcr2s (atom nil) - - u1d (atom nil) - u2d (atom nil) - u3d (atom nil) - pulse-d (atom nil) - pc1d (atom nil)] - (ts/with-source-and-dest-dbs - (testing "serializing the pulse, channel and recipients" - (ts/with-source-db - (reset! u1s (ts/create! User :first_name "Alex" :last_name "Lifeson" :email "alifeson@rush.yyz")) - (reset! u2s (ts/create! User :first_name "Geddy" :last_name "Lee" :email "glee@rush.yyz")) - (reset! u3s (ts/create! User :first_name "Neil" :last_name "Peart" :email "neil@rush.yyz")) - (reset! pulse-s (ts/create! Pulse :name "Heartbeat" :creator_id (:id @u1s))) - (reset! pc1s (ts/create! PulseChannel - :pulse_id (:id @pulse-s) - :channel_type :email - :schedule_type :daily - :schedule_hour 16)) - (reset! pc2s (ts/create! PulseChannel - :pulse_id (:id @pulse-s) - :channel_type :slack - :schedule_type :hourly)) - ;; Only Lifeson and Lee are recipients in the source. - (reset! pcr1s (ts/create! PulseChannelRecipient :pulse_channel_id (:id @pc1s) :user_id (:id @u1s))) - (reset! pcr2s (ts/create! PulseChannelRecipient :pulse_channel_id (:id @pc1s) :user_id (:id @u2s))) - (reset! serialized (into [] (serdes.extract/extract-metabase {}))))) - - (testing "recipients are serialized as :recipients [email] on the PulseChannel" - (is (= #{["alifeson@rush.yyz" "glee@rush.yyz"] - []} - (set (map :recipients (by-model @serialized "PulseChannel")))))) - - (testing "deserialization merges the existing recipients with the new ones" - (ts/with-dest-db - ;; Users in a different order, so different IDs. - (reset! u2d (ts/create! User :first_name "Geddy" :last_name "Lee" :email "glee@rush.yyz")) - (reset! u1d (ts/create! User :first_name "Alex" :last_name "Lifeson" :email "alifeson@rush.yyz")) - (reset! u3d (ts/create! User :first_name "Neil" :last_name "Peart" :email "neil@rush.yyz")) - (reset! pulse-d (ts/create! Pulse :name "Heartbeat" :creator_id (:id @u1d) :entity_id (:entity_id @pulse-s))) - (reset! pc1d (ts/create! PulseChannel - :entity_id (:entity_id @pc1s) - :pulse_id (:id @pulse-d) - :channel_type :email - :schedule_type :daily - :schedule_hour 16)) - ;; Only Lee and Peart are recipients in the source. - (ts/create! PulseChannelRecipient :pulse_channel_id (:id @pc1d) :user_id (:id @u2d)) - (ts/create! PulseChannelRecipient :pulse_channel_id (:id @pc1d) :user_id (:id @u3d)) - - (is (= 2 (db/count PulseChannelRecipient))) - (is (= #{(:id @u2d) (:id @u3d)} - (db/select-field :user_id PulseChannelRecipient))) - - (serdes.load/load-metabase (ingestion-in-memory @serialized)) - - (is (= 3 (db/count PulseChannelRecipient))) - (is (= #{(:id @u1d) (:id @u2d) (:id @u3d)} - (db/select-field :user_id PulseChannelRecipient))))))))) - (deftest card-dataset-query-test ;; Card.dataset_query is a JSON-encoded MBQL query, which contain database, table, and field IDs - these need to be ;; converted to a portable form and read back in. @@ -580,18 +488,19 @@ :target [:dimension [:field (:id @field1s) {:source-field (:id @field2s)}]]}])) (reset! serialized (into [] (serdes.extract/extract-metabase {}))) - (let [card (-> @serialized (by-model "Card") first) - dashcard (-> @serialized (by-model "DashboardCard") first)] + (let [card (-> @serialized (by-model "Card") first) + dash (-> @serialized (by-model "Dashboard") first)] (testing "exported :parameter_mappings are properly converted" (is (= [{:parameter_id "12345678" :target [:dimension [:field ["my-db" nil "orders" "subtotal"] {:source-field ["my-db" nil "orders" "invoice"]}]]}] (:parameter_mappings card))) - (is (= [{:parameter_id "deadbeef" - :card_id (:entity_id @card1s) - :target [:dimension [:field ["my-db" nil "orders" "subtotal"] - {:source-field ["my-db" nil "orders" "invoice"]}]]}] - (:parameter_mappings dashcard)))) + (is (schema= [{:parameter_mappings [{:parameter_id (s/eq "deadbeef") + :card_id (s/eq (:entity_id @card1s)) + :target (s/eq [:dimension [:field ["my-db" nil "orders" "subtotal"] + {:source-field ["my-db" nil "orders" "invoice"]}]])}] + s/Keyword s/Any}] + (:ordered_cards dash)))) (testing "exported :visualization_settings are properly converted" (let [expected {:table.pivot_column "SOURCE" @@ -614,8 +523,7 @@ (is (= expected (:visualization_settings card))) (is (= expected - (:visualization_settings dashcard)))))))) - + (-> dash :ordered_cards first :visualization_settings)))))))) (testing "deserializing adjusts the IDs properly" @@ -660,6 +568,116 @@ :target [:dimension [:field (:id @field1d) {:source-field (:id @field2d)}]]}] (:parameter_mappings @dashcard1d)))))))))) +(deftest timelines-test + (testing "timelines" + (let [serialized (atom nil) + coll1s (atom nil) + user1s (atom nil) + timeline1s (atom nil) + event1s (atom nil) + event2s (atom nil) + timeline2s (atom nil) + event3s (atom nil) + + coll1d (atom nil) + user1d (atom nil) + timeline1d (atom nil) + timeline2d (atom nil) + eventsT1 (atom nil) + eventsT2 (atom nil)] + + (ts/with-source-and-dest-dbs + (testing "serialize correctly" + (ts/with-source-db + (reset! coll1s (ts/create! Collection :name "col1")) + (reset! user1s (ts/create! User :first_name "Tom" :last_name "Scholz" :email "tom@bost.on")) + (reset! timeline1s (ts/create! Timeline :name "Some events" :creator_id (:id @user1s) + :collection_id (:id @coll1s))) + (reset! event1s (ts/create! TimelineEvent :name "First thing" :timeline_id (:id @timeline1s) + :creator_id (:id @user1s) :timezone "America/New_York" + :timestamp (t/local-date 2022 11 3))) + (reset! event2s (ts/create! TimelineEvent :name "Second thing" :timeline_id (:id @timeline1s) + :creator_id (:id @user1s) :timezone "America/New_York" + :timestamp (t/local-date 2022 11 8))) + (reset! timeline2s (ts/create! Timeline :name "More events" :creator_id (:id @user1s) + :collection_id (:id @coll1s))) + (reset! event3s (ts/create! TimelineEvent :name "Different event" :timeline_id (:id @timeline2s) + :creator_id (:id @user1s) :timezone "America/New_York" + :time_matters true :timestamp (t/offset-date-time 2022 10 31 19 00 00))) + + (testing "expecting 3 events" + (is (= 3 (db/count TimelineEvent)))) + + (reset! serialized (into [] (serdes.extract/extract-metabase {}))) + + (let [timelines (by-model @serialized "Timeline") + timeline1 (first (filter #(= (:entity_id %) (:entity_id @timeline1s)) timelines)) + timeline2 (first (filter #(= (:entity_id %) (:entity_id @timeline2s)) timelines))] + (testing "with inline :events" + (is (schema= {:serdes/meta (s/eq [{:model "Timeline" + :id (:entity_id timeline1) + :label "some_events"}]) + :archived (s/eq false) + :collection_id (s/eq (:entity_id @coll1s)) + :name (s/eq "Some events") + :creator_id (s/eq "tom@bost.on") + (s/optional-key :updated_at) OffsetDateTime + :created_at OffsetDateTime + :entity_id (s/eq (:entity_id timeline1)) + (s/optional-key :icon) (s/maybe s/Str) + :description (s/maybe s/Str) + (s/optional-key :default) s/Bool + :events [{:timezone s/Str + :time_matters s/Bool + :name s/Str + :archived s/Bool + :description (s/maybe s/Str) + :creator_id s/Str + (s/optional-key :icon) (s/maybe s/Str) + :created_at OffsetDateTime + (s/optional-key :updated_at) OffsetDateTime + :timestamp s/Str}]} + timeline1)) + (is (= 2 (-> timeline1 :events count))) + (is (= 1 (-> timeline2 :events count))))))) + + (testing "deserializing merges events properly" + (ts/with-dest-db + ;; The collection, timeline 1 and event 2 already exist. Event 1, plus timeline 2 and its event 3, are new. + (reset! user1d (ts/create! User :first_name "Tom" :last_name "Scholz" :email "tom@bost.on")) + (reset! coll1d (ts/create! Collection :name "col1" :entity_id (:entity_id @coll1s))) + (reset! timeline1d (ts/create! Timeline :name "Some events" :creator_id (:id @user1s) + :entity_id (:entity_id @timeline1s) + :collection_id (:id @coll1d))) + (ts/create! TimelineEvent :name "Second thing with different name" :timeline_id (:id @timeline1s) + :timestamp (:timestamp @event2s) + :creator_id (:id @user1s) :timezone "America/New_York") + + ;; Load the serialized content. + (serdes.load/load-metabase (ingestion-in-memory @serialized)) + + ;; Fetch the relevant bits + (reset! timeline2d (db/select-one Timeline :entity_id (:entity_id @timeline2s))) + (reset! eventsT1 (db/select TimelineEvent :timeline_id (:id @timeline1d))) + (reset! eventsT2 (db/select TimelineEvent :timeline_id (:id @timeline2d))) + + (testing "no duplication - there are two timelines with the right event counts" + (is (some? @timeline2d)) + (is (= 2 (count @eventsT1))) + (is (= 1 (count @eventsT2)))) + + (testing "resulting events match up" + (let [[event1 event2] (sort-by :timestamp @eventsT1)] + (is (= (:timestamp @event1s) (:timestamp event1))) + (is (= (:timestamp @event2s) (:timestamp event2))) + + (is (= (:timestamp @event3s) + (:timestamp (first @eventsT2)))) + + (is (= (:name @event2s) + (:name event2)) + "existing event name should be updated"))))))))) + (deftest users-test ;; Users are serialized as their email address. If a corresponding user is found during deserialization, its ID is ;; used. However, if no such user exists, a new one is created with mostly blank fields. @@ -817,7 +835,93 @@ (is (= 2 (db/count FieldValues :field_id [:in fields]))))) (testing "existing FieldValues are properly found and updated" - (is (= (:values @fv1s) (:values @fv1d)))) + (is (= (set (:values @fv1s)) (set (:values @fv1d))))) (testing "new FieldValues are properly added" (is (= (dissoc @fv2s :id :field_id :created_at :updated_at) (dissoc @fv2d :id :field_id :created_at :updated_at))))))))) + +(deftest bare-import-test + ;; If the dependencies of an entity exist in the receiving database, they don't need to be in the export. + ;; This tests that such an import will succeed, and that it still fails when the dependency is not found in + ;; either location. + (let [db1s (atom nil) + table1s (atom nil)] + + (testing "loading a bare card" + (ts/with-empty-h2-app-db + (reset! db1s (ts/create! Database :name "my-db")) + (reset! table1s (ts/create! Table :name "CUSTOMERS" :db_id (:id @db1s))) + (ts/create! Field :name "STATE" :table_id (:id @table1s)) + (ts/create! User :first_name "Geddy" :last_name "Lee" :email "glee@rush.yyz") + + (testing "depending on existing values works" + (let [ingestion (ingestion-in-memory [{:serdes/meta [{:model "Card" :id "0123456789abcdef_0123"}] + :created_at (t/instant) + :creator_id "glee@rush.yyz" + :database_id "my-db" + :dataset_query {:database "my-db" + :type :query + :query {:source-table ["my-db" nil "CUSTOMERS"]}} + :display :table + :entity_id "0123456789abcdef_0123" + :name "Some card" + :table_id ["my-db" nil "CUSTOMERS"] + :visualization_settings {}}])] + (is (some? (serdes.load/load-metabase ingestion))))) + + (testing "depending on nonexisting values fails" + (let [ingestion (ingestion-in-memory [{:serdes/meta [{:model "Card" :id "0123456789abcdef_0123"}] + :created_at (t/instant) + :creator_id "glee@rush.yyz" + :database_id "bad-db" + :dataset_query {:database "bad-db" + :type :query + :query {:source-table ["bad-db" nil "CUSTOMERS"]}} + :display :table + :entity_id "0123456789abcdef_0123" + :name "Some card" + :table_id ["bad-db" nil "CUSTOMERS"] + :visualization_settings {}}])] + (is (thrown-with-msg? clojure.lang.ExceptionInfo + #"Failed to read file" + (serdes.load/load-metabase ingestion))))))))) + +(deftest card-with-snippet-test + (let [db1s (atom nil) + table1s (atom nil) + snippet1s (atom nil) + card1s (atom nil) + extracted (atom nil)] + (testing "snippets referenced by native cards must be deserialized" + (ts/with-empty-h2-app-db + (reset! db1s (ts/create! Database :name "my-db")) + (reset! table1s (ts/create! Table :name "CUSTOMERS" :db_id (:id @db1s))) + (reset! snippet1s (ts/create! NativeQuerySnippet :name "some snippet")) + (reset! card1s (ts/create! Card + :name "the query" + :dataset_query {:database (:id @db1s) + :native {:template-tags {"snippet: things" + {:id "e2d15f07-37b3-01fc-3944-2ff860a5eb46", + :name "snippet: filtered data", + :display-name "Snippet: Filtered Data", + :type :snippet, + :snippet-name "filtered data", + :snippet-id (:id @snippet1s)}}}})) + (ts/create! User :first_name "Geddy" :last_name "Lee" :email "glee@rush.yyz") + + (testing "on extraction" + (reset! extracted (serdes.base/extract-one "Card" {} @card1s)) + (is (= (:entity_id @snippet1s) + (-> @extracted :dataset_query :native :template-tags (get "snippet: things") :snippet-id)))) + + (testing "when loading" + (let [new-eid (u/generate-nano-id) + ingestion (ingestion-in-memory [(assoc @extracted :entity_id new-eid)])] + (is (some? (serdes.load/load-metabase ingestion))) + (is (= (:id @snippet1s) + (-> (db/select-one Card :entity_id new-eid) + :dataset_query + :native + :template-tags + (get "snippet: things") + :snippet-id))))))))) diff --git a/enterprise/backend/test/metabase_enterprise/serialization/v2/storage/yaml_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/v2/storage/yaml_test.clj index 8542a969b36c..9b6b3560a7a9 100644 --- a/enterprise/backend/test/metabase_enterprise/serialization/v2/storage/yaml_test.clj +++ b/enterprise/backend/test/metabase_enterprise/serialization/v2/storage/yaml_test.clj @@ -6,18 +6,18 @@ [metabase-enterprise.serialization.test-util :as ts] [metabase-enterprise.serialization.v2.extract :as extract] [metabase-enterprise.serialization.v2.storage.yaml :as storage.yaml] - [metabase.models :refer [Collection Database Field FieldValues Table]] + [metabase.models :refer [Card Collection Dashboard Database Field FieldValues NativeQuerySnippet Table]] [metabase.models.serialization.base :as serdes.base] [metabase.util.date-2 :as u.date] [toucan.db :as db] [yaml.core :as yaml])) -(defn- dir->file-set [dir] - (->> dir - .listFiles - (filter #(.isFile %)) - (map #(.getName %)) - set)) +(defn- file-set [dir] + (let [base (.toPath dir)] + (set (for [file (file-seq dir) + :when (.isFile file) + :let [rel (.relativize base (.toPath file))]] + (mapv str rel))))) (deftest basic-dump-test (ts/with-random-dump-dir [dump-dir "serdesv2-"] @@ -25,15 +25,16 @@ (ts/with-temp-dpc [Collection [parent {:name "Some Collection"}] Collection [child {:name "Child Collection" :location (format "/%d/" (:id parent))}]] (let [export (into [] (extract/extract-metabase nil)) - parent-filename (format "%s+some_collection.yaml" (:entity_id parent)) - child-filename (format "%s+child_collection.yaml" (:entity_id child))] + parent-filename (format "%s_some_collection" (:entity_id parent)) + child-filename (format "%s_child_collection" (:entity_id child))] (storage.yaml/store! export dump-dir) (testing "the right files in the right places" - (is (= #{parent-filename child-filename} - (dir->file-set (io/file dump-dir "Collection"))) - "Entities go in subdirectories") - (is (= #{"settings.yaml"} - (dir->file-set (io/file dump-dir))) + (is (= #{[parent-filename (str parent-filename ".yaml")] + [parent-filename child-filename (str child-filename ".yaml")]} + (file-set (io/file dump-dir "collections"))) + "collections form a tree, with same-named files") + (is (contains? (file-set (io/file dump-dir)) + ["settings.yaml"]) "A few top-level files are expected")) (testing "the Collections properly exported" @@ -41,42 +42,107 @@ (dissoc :id :location) (assoc :parent_id nil) (update :created_at t/offset-date-time)) - (-> (yaml/from-file (io/file dump-dir "Collection" parent-filename)) + (-> (yaml/from-file (io/file dump-dir "collections" parent-filename (str parent-filename ".yaml"))) (update :created_at t/offset-date-time)))) (is (= (-> (into {} (db/select-one Collection :id (:id child))) (dissoc :id :location) (assoc :parent_id (:entity_id parent)) (update :created_at t/offset-date-time)) - (-> (yaml/from-file (io/file dump-dir "Collection" child-filename)) + (-> (yaml/from-file (io/file dump-dir "collections" parent-filename + child-filename (str child-filename ".yaml"))) (update :created_at t/offset-date-time)))))))))) +(deftest collection-nesting-test + (ts/with-random-dump-dir [dump-dir "serdesv2-"] + (ts/with-empty-h2-app-db + (ts/with-temp-dpc [Collection [grandparent {:name "Grandparent Collection" + :location "/"}] + Collection [parent {:name "Parent Collection" + :location (str "/" (:id grandparent) "/")}] + Collection [child {:name "Child Collection" + :location (str "/" (:id grandparent) "/" (:id parent) "/")}] + Card [c1 {:name "root card" :collection_id nil}] + Card [c2 {:name "grandparent card" :collection_id (:id grandparent)}] + Card [c3 {:name "parent card" :collection_id (:id parent)}] + Card [c4 {:name "child card" :collection_id (:id child)}] + Dashboard [d1 {:name "parent dash" :collection_id (:id parent)}]] + (let [export (into [] (extract/extract-metabase nil))] + (storage.yaml/store! export dump-dir) + (testing "the right files in the right places" + (let [gp-dir (str (:entity_id grandparent) "_grandparent_collection") + p-dir (str (:entity_id parent) "_parent_collection") + c-dir (str (:entity_id child) "_child_collection")] + (is (= #{[gp-dir (str gp-dir ".yaml")] ; Grandparent collection + [gp-dir p-dir (str p-dir ".yaml")] ; Parent collection + [gp-dir p-dir c-dir (str c-dir ".yaml")] ; Child collection + ["cards" (str (:entity_id c1) "_root_card.yaml")] ; Root card + [gp-dir "cards" (str (:entity_id c2) "_grandparent_card.yaml")] ; Grandparent card + [gp-dir p-dir "cards" (str (:entity_id c3) "_parent_card.yaml")] ; Parent card + [gp-dir p-dir c-dir "cards" (str (:entity_id c4) "_child_card.yaml")] ; Child card + [gp-dir p-dir "dashboards" (str (:entity_id d1) "_parent_dash.yaml")]} ; Parent dashboard + (file-set (io/file dump-dir "collections"))))))))))) + +(deftest snippets-collections-nesting-test + (ts/with-random-dump-dir [dump-dir "serdesv2-"] + (ts/with-empty-h2-app-db + (ts/with-temp-dpc [Collection [grandparent {:name "Grandparent Collection" + :namespace :snippets + :location "/"}] + Collection [parent {:name "Parent Collection" + :namespace :snippets + :location (str "/" (:id grandparent) "/")}] + Collection [child {:name "Child Collection" + :namespace :snippets + :location (str "/" (:id grandparent) "/" (:id parent) "/")}] + NativeQuerySnippet [c1 {:name "root snippet" :collection_id nil}] + NativeQuerySnippet [c2 {:name "grandparent snippet" :collection_id (:id grandparent)}] + NativeQuerySnippet [c3 {:name "parent snippet" :collection_id (:id parent)}] + NativeQuerySnippet [c4 {:name "child snippet" :collection_id (:id child)}]] + (let [export (into [] (extract/extract-metabase nil))] + (storage.yaml/store! export dump-dir) + (let [gp-dir (str (:entity_id grandparent) "_grandparent_collection") + p-dir (str (:entity_id parent) "_parent_collection") + c-dir (str (:entity_id child) "_child_collection")] + (testing "collections under collections/" + (is (= #{[gp-dir (str gp-dir ".yaml")] ; Grandparent collection + [gp-dir p-dir (str p-dir ".yaml")] ; Parent collection + [gp-dir p-dir c-dir (str c-dir ".yaml")]} ; Child collection + (file-set (io/file dump-dir "collections"))))) + (testing "snippets under snippets/" + (is (= #{ + [(str (:entity_id c1) "_root_snippet.yaml")] ; Root snippet + [gp-dir (str (:entity_id c2) "_grandparent_snippet.yaml")] ; Grandparent snippet + [gp-dir p-dir (str (:entity_id c3) "_parent_snippet.yaml")] ; Parent snippet + [gp-dir p-dir c-dir (str (:entity_id c4) "_child_snippet.yaml")]} ; Child snippet + (file-set (io/file dump-dir "snippets"))))))))))) + (deftest embedded-slash-test (ts/with-random-dump-dir [dump-dir "serdesv2-"] (ts/with-empty-h2-app-db (ts/with-temp-dpc [Database [db {:name "My Company Data"}] Table [table {:name "Customers" :db_id (:id db)}] Field [website {:name "Company/organization website" :table_id (:id table)}] - FieldValues [_ {:field_id (:id website)}]] + FieldValues [_ {:field_id (:id website)}] + Table [_ {:name "Orders/Invoices" :db_id (:id db)}]] (let [export (into [] (extract/extract-metabase nil))] (storage.yaml/store! export dump-dir) (testing "the right files in the right places" - (is (= #{"Company__SLASH__organization website.yaml"} - (dir->file-set (io/file dump-dir "Database" "My Company Data" "Table" "Customers" "Field"))) + (is (= #{["Company__SLASH__organization website.yaml"] + ["Company__SLASH__organization website___fieldvalues.yaml"]} + (file-set (io/file dump-dir "databases" "My Company Data" "tables" "Customers" "fields"))) "Slashes in file names get escaped") - (is (= #{"0.yaml"} - (dir->file-set (io/file dump-dir "Database" "My Company Data" "Table" "Customers" - "Field" "Company__SLASH__organization website" - "FieldValues"))) - "Slashes in parent directory names get escaped")) + (is (contains? (file-set (io/file dump-dir "databases" "My Company Data" "tables")) + ["Orders__SLASH__Invoices" "Orders__SLASH__Invoices.yaml"]) + "Slashes in directory names get escaped")) (testing "the Field was properly exported" (is (= (-> (into {} (serdes.base/extract-one "Field" {} (db/select-one 'Field :id (:id website)))) (update :created_at u.date/format) (dissoc :serdes/meta)) (-> (yaml/from-file (io/file dump-dir - "Database" "My Company Data" - "Table" "Customers" - "Field" "Company__SLASH__organization website.yaml")) + "databases" "My Company Data" + "tables" "Customers" + "fields" "Company__SLASH__organization website.yaml")) (update :visibility_type keyword) (update :base_type keyword)))))))))) diff --git a/enterprise/frontend/src/metabase-enterprise/audit_app/lib/mode.js b/enterprise/frontend/src/metabase-enterprise/audit_app/lib/mode.js index 42513af79796..4de286d2afe8 100644 --- a/enterprise/frontend/src/metabase-enterprise/audit_app/lib/mode.js +++ b/enterprise/frontend/src/metabase-enterprise/audit_app/lib/mode.js @@ -88,5 +88,5 @@ const AuditDrill = ({ question, clicked }) => { export const AuditMode = { name: "audit", - drills: () => [AuditDrill], + drills: [AuditDrill], }; diff --git a/enterprise/frontend/src/metabase-enterprise/auth/components/SettingsJWTForm/SettingsJWTForm.jsx b/enterprise/frontend/src/metabase-enterprise/auth/components/SettingsJWTForm/SettingsJWTForm.jsx index 13f20aaa5284..5dbcdbd3c3c9 100644 --- a/enterprise/frontend/src/metabase-enterprise/auth/components/SettingsJWTForm/SettingsJWTForm.jsx +++ b/enterprise/frontend/src/metabase-enterprise/auth/components/SettingsJWTForm/SettingsJWTForm.jsx @@ -1,4 +1,4 @@ -import React from "react"; +import React, { useCallback } from "react"; import PropTypes from "prop-types"; import { connect } from "react-redux"; import { t } from "ttag"; @@ -8,15 +8,18 @@ import { FormButton } from "./SettingsJWTForm.styled"; const propTypes = { settingValues: PropTypes.object.isRequired, - updateSettings: PropTypes.func.isRequired, + onSubmit: PropTypes.func.isRequired, }; -const SettingsJWTForm = ({ settingValues, updateSettings, ...props }) => { +const SettingsJWTForm = ({ settingValues, onSubmit, ...props }) => { const isEnabled = settingValues["jwt-enabled"]; - const handleAutoEnableSubmit = formData => { - return updateSettings({ ...formData, "jwt-enabled": true }); - }; + const handleSubmit = useCallback( + values => { + return onSubmit({ ...values, "jwt-enabled": true }); + }, + [onSubmit], + ); return ( { layout={FORM_LAYOUT} breadcrumbs={BREADCRUMBS} settingValues={settingValues} - updateSettings={updateSettings} - renderSubmitButton={ - !isEnabled && - (({ disabled, pristine, onSubmit }) => ( - onSubmit(handleAutoEnableSubmit)} - normalText={t`Save and enable`} - successText={t`Changes saved!`} - /> - )) - } - renderExtraButtons={ - !isEnabled && - (({ disabled, pristine, onSubmit }) => ( - onSubmit(updateSettings)} - normalText={t`Save but don't enable`} - successText={t`Changes saved!`} - /> - )) - } + updateSettings={handleSubmit} + renderSubmitButton={({ disabled, pristine, onSubmit }) => ( + + )} /> ); }; @@ -80,7 +69,7 @@ const BREADCRUMBS = [ ]; const mapDispatchToProps = { - updateSettings, + onSubmit: updateSettings, }; export default connect(null, mapDispatchToProps)(SettingsJWTForm); diff --git a/enterprise/frontend/src/metabase-enterprise/auth/components/SettingsSAMLForm.jsx b/enterprise/frontend/src/metabase-enterprise/auth/components/SettingsSAMLForm.jsx deleted file mode 100644 index 22110357f3b8..000000000000 --- a/enterprise/frontend/src/metabase-enterprise/auth/components/SettingsSAMLForm.jsx +++ /dev/null @@ -1,189 +0,0 @@ -/* eslint-disable react/prop-types */ -import React, { Component } from "react"; -import { connect } from "react-redux"; -import { t } from "ttag"; -import _ from "underscore"; - -import { updateSettings } from "metabase/admin/settings/settings"; -import { settingToFormField } from "metabase/admin/settings/utils"; - -import Form, { - FormField, - FormSubmit, - FormMessage, - FormSection, -} from "metabase/containers/FormikForm"; - -import Breadcrumbs from "metabase/components/Breadcrumbs"; -import CopyWidget from "metabase/components/CopyWidget"; - -import GroupMappingsWidget from "metabase/admin/settings/components/widgets/GroupMappingsWidget"; - -import MetabaseSettings from "metabase/lib/settings"; -import { SAMLFormSection } from "./SettingsSAMLForm.styled"; - -class SettingsSAMLForm extends Component { - render() { - const { elements, settingValues, updateSettings } = this.props; - // TODO: move these to an outer component so we don't have to do it in every form page - const setting = name => - _.findWhere(elements, { key: name }) || { key: name }; - const settingField = name => settingToFormField(setting(name)); - - const initialValues = { ...settingValues }; - - // HACK: this is to make the default show up as selectable text instead of placeholder - const addDefaultAsInitialValue = name => { - if (initialValues[name] == null && setting(name).default != null) { - initialValues[name] = setting(name).default; - } - }; - addDefaultAsInitialValue("saml-attribute-email"); - addDefaultAsInitialValue("saml-attribute-firstname"); - addDefaultAsInitialValue("saml-attribute-lastname"); - - const acsConsumerUrl = MetabaseSettings.get("site-url") + "/auth/sso"; - - return ( - - -

{t`Set up SAML-based SSO`}

- - -

{t`Configure your identity provider (IdP)`}

-

{t`Your identity provider will need the following info about Metabase.`}

- -
-
{t`URL the IdP should redirect back to`}
-
{t`This is called the Single Sign On URL in Okta, the Application Callback URL in Auth0, - and the ACS (Consumer) URL in OneLogin. `}
- -
- -

{t`SAML attributes`}

-

{t`In most IdPs, you'll need to put each of these in an input box labeled - "Name" in the attribute statements section.`}

- - } - /> - } - /> - } - /> - - - -

{t`Tell Metabase about your identity provider`}

-

{t`Metabase will need the following info about your provider.`}

- - - - - - - - - - - - - - - -

{t`Synchronize group membership with your SSO`}

-

- {t`To enable this, you'll need to create mappings to tell Metabase which group(s) your users should - be added to based on the SSO group they're in.`} -

- ( - - updateSettings({ [key]: value }) - } - mappingSetting="saml-group-mappings" - groupHeading={t`Group Name`} - groupPlaceholder={t`Group Name`} - /> - )} - /> - - - -
- -
-
- {t`Save changes`} -
- - ); - } -} - -export default connect(null, { updateSettings })(SettingsSAMLForm); diff --git a/enterprise/frontend/src/metabase-enterprise/auth/components/SettingsSAMLForm/SettingsSAMLForm.jsx b/enterprise/frontend/src/metabase-enterprise/auth/components/SettingsSAMLForm/SettingsSAMLForm.jsx new file mode 100644 index 000000000000..1125d1d67eaa --- /dev/null +++ b/enterprise/frontend/src/metabase-enterprise/auth/components/SettingsSAMLForm/SettingsSAMLForm.jsx @@ -0,0 +1,219 @@ +import React, { useCallback, useMemo } from "react"; +import { connect } from "react-redux"; +import PropTypes from "prop-types"; +import { jt, t } from "ttag"; +import _ from "underscore"; +import Breadcrumbs from "metabase/components/Breadcrumbs"; +import CopyWidget from "metabase/components/CopyWidget"; +import ExternalLink from "metabase/core/components/ExternalLink"; +import Form, { + FormField, + FormSubmit, + FormMessage, + FormSection, +} from "metabase/containers/FormikForm"; +import MetabaseSettings from "metabase/lib/settings"; +import GroupMappingsWidget from "metabase/admin/settings/components/widgets/GroupMappingsWidget"; +import { updateSettings } from "metabase/admin/settings/settings"; +import { settingToFormField } from "metabase/admin/settings/utils"; +import { + SAMLFormCaption, + SAMLFormFooter, + SAMLFormSection, +} from "./SettingsSAMLForm.styled"; + +const propTypes = { + elements: PropTypes.array, + settingValues: PropTypes.object, + onSubmit: PropTypes.func, +}; + +const SettingsSAMLForm = ({ elements = [], settingValues = {}, onSubmit }) => { + const isEnabled = Boolean(settingValues["saml-enabled"]); + + const settings = useMemo(() => { + return _.indexBy(elements, "key"); + }, [elements]); + + const fields = useMemo(() => { + return _.mapObject(settings, settingToFormField); + }, [settings]); + + const defaultValues = useMemo(() => { + return _.mapObject(settings, "default"); + }, [settings]); + + const attributeValues = useMemo(() => { + return getAttributeValues(settingValues, defaultValues); + }, [settingValues, defaultValues]); + + const handleSubmit = useCallback( + values => onSubmit({ ...values, "saml-enabled": true }), + [onSubmit], + ); + + return ( +
+ +

{t`Set up SAML-based SSO`}

+ + {jt`Use the settings below to configure your SSO via SAML. If you have any questions, check out our ${( + {t`documentation`} + )}.`} + + +

{t`Configure your identity provider (IdP)`}

+

{t`Your identity provider will need the following info about Metabase.`}

+ +
+
{t`URL the IdP should redirect back to`}
+
{t`This is called the Single Sign On URL in Okta, the Application Callback URL in Auth0, + and the ACS (Consumer) URL in OneLogin. `}
+ +
+ +

{t`SAML attributes`}

+

{t`In most IdPs, you'll need to put each of these in an input box labeled + "Name" in the attribute statements section.`}

+ + } + /> + } + /> + } + /> + + + +

{t`Tell Metabase about your identity provider`}

+

{t`Metabase will need the following info about your provider.`}

+ + + + + + + + + + + + + + + +

{t`Synchronize group membership with your SSO`}

+

+ {t`To enable this, you'll need to create mappings to tell Metabase which group(s) your users should + be added to based on the SSO group they're in.`} +

+ ( + onSubmit({ [key]: value })} + mappingSetting="saml-group-mappings" + groupHeading={t`Group Name`} + groupPlaceholder={t`Group Name`} + /> + )} + /> + + + +
+ +
+ + + {isEnabled ? t`Save changes` : t`Save and enable`} + + + + ); +}; + +const SAML_ATTRS = [ + "saml-attribute-email", + "saml-attribute-firstname", + "saml-attribute-lastname", +]; + +const getAttributeValues = (values, defaults) => { + return _.object(SAML_ATTRS.map(key => [key, values[key] ?? defaults[key]])); +}; + +const getAcsCustomerUrl = () => { + return `${MetabaseSettings.get("site-url")}/auth/sso`; +}; + +const getDocsUrl = () => { + return MetabaseSettings.docsUrl("people-and-groups/authenticating-with-saml"); +}; + +SettingsSAMLForm.propTypes = propTypes; + +const mapDispatchToProps = { + onSubmit: updateSettings, +}; + +export default connect(null, mapDispatchToProps)(SettingsSAMLForm); diff --git a/enterprise/frontend/src/metabase-enterprise/auth/components/SettingsSAMLForm.styled.tsx b/enterprise/frontend/src/metabase-enterprise/auth/components/SettingsSAMLForm/SettingsSAMLForm.styled.tsx similarity index 67% rename from enterprise/frontend/src/metabase-enterprise/auth/components/SettingsSAMLForm.styled.tsx rename to enterprise/frontend/src/metabase-enterprise/auth/components/SettingsSAMLForm/SettingsSAMLForm.styled.tsx index 9d76d6a8163c..3e56ec53bff7 100644 --- a/enterprise/frontend/src/metabase-enterprise/auth/components/SettingsSAMLForm.styled.tsx +++ b/enterprise/frontend/src/metabase-enterprise/auth/components/SettingsSAMLForm/SettingsSAMLForm.styled.tsx @@ -11,3 +11,13 @@ export const SAMLFormSection = styled.div` border: 1px solid ${color("border")}; border-radius: 0.5rem; `; + +export const SAMLFormCaption = styled.div` + color: ${color("text-medium")}; + margin-bottom: 1rem; +`; + +export const SAMLFormFooter = styled.div` + display: flex; + gap: 0.5rem; +`; diff --git a/enterprise/frontend/src/metabase-enterprise/auth/components/SettingsSAMLForm/index.js b/enterprise/frontend/src/metabase-enterprise/auth/components/SettingsSAMLForm/index.js new file mode 100644 index 000000000000..1db0fcff3058 --- /dev/null +++ b/enterprise/frontend/src/metabase-enterprise/auth/components/SettingsSAMLForm/index.js @@ -0,0 +1 @@ +export { default } from "./SettingsSAMLForm"; diff --git a/enterprise/frontend/src/metabase-enterprise/auth/constants.ts b/enterprise/frontend/src/metabase-enterprise/auth/constants.ts new file mode 100644 index 000000000000..d5505d3b1909 --- /dev/null +++ b/enterprise/frontend/src/metabase-enterprise/auth/constants.ts @@ -0,0 +1,29 @@ +import * as Yup from "yup"; + +export const JWT_SCHEMA = Yup.object({ + "jwt-enabled": Yup.boolean().default(false), + "jwt-identity-provider-uri": Yup.string().nullable().default(null), + "jwt-shared-secret": Yup.string().nullable().default(null), + "jwt-attribute-email": Yup.string().nullable().default(null), + "jwt-attribute-firstname": Yup.string().nullable().default(null), + "jwt-attribute-lastname": Yup.string().nullable().default(null), + "jwt-group-sync": Yup.boolean().default(false), + "jwt-group-mappings": Yup.object().default(null), +}); + +export const SAML_SCHEMA = Yup.object({ + "saml-enabled": Yup.boolean().default(false), + "saml-identity-provider-uri": Yup.string().nullable().default(null), + "saml-identity-provider-issuer": Yup.string().nullable().default(null), + "saml-identity-provider-certificate": Yup.string().nullable().default(null), + "saml-application-name": Yup.string().nullable().default(null), + "saml-keystore-path": Yup.string().nullable().default(null), + "saml-keystore-password": Yup.string().nullable().default(null), + "saml-keystore-alias": Yup.string().nullable().default(null), + "saml-attribute-email": Yup.string().nullable().default(null), + "saml-attribute-firstname": Yup.string().nullable().default(null), + "saml-attribute-lastname": Yup.string().nullable().default(null), + "saml-attribute-group": Yup.string().nullable().default(null), + "saml-group-sync": Yup.boolean().default(false), + "saml-group-mappings": Yup.object().default(null), +}); diff --git a/enterprise/frontend/src/metabase-enterprise/auth/containers/JwtAuthCard/JwtAuthCard.tsx b/enterprise/frontend/src/metabase-enterprise/auth/containers/JwtAuthCard/JwtAuthCard.tsx new file mode 100644 index 000000000000..057013beef90 --- /dev/null +++ b/enterprise/frontend/src/metabase-enterprise/auth/containers/JwtAuthCard/JwtAuthCard.tsx @@ -0,0 +1,20 @@ +import { t } from "ttag"; +import { connect } from "react-redux"; +import { getSetting } from "metabase/selectors/settings"; +import { updateSettings } from "metabase/admin/settings/settings"; +import AuthCard from "metabase/admin/settings/auth/components/AuthCard"; +import { Dispatch, State } from "metabase-types/store"; +import { JWT_SCHEMA } from "../../constants"; + +const mapStateToProps = (state: State) => ({ + type: "jwt", + name: t`JWT`, + description: t`Allows users to login via a JWT Identity Provider.`, + isConfigured: getSetting(state, "jwt-configured"), +}); + +const mapDispatchToProps = (dispatch: Dispatch) => ({ + onDeactivate: () => dispatch(updateSettings(JWT_SCHEMA.getDefault())), +}); + +export default connect(mapStateToProps, mapDispatchToProps)(AuthCard); diff --git a/enterprise/frontend/src/metabase-enterprise/auth/containers/JwtAuthCard/index.ts b/enterprise/frontend/src/metabase-enterprise/auth/containers/JwtAuthCard/index.ts new file mode 100644 index 000000000000..958122a07aca --- /dev/null +++ b/enterprise/frontend/src/metabase-enterprise/auth/containers/JwtAuthCard/index.ts @@ -0,0 +1 @@ +export { default } from "./JwtAuthCard"; diff --git a/enterprise/frontend/src/metabase-enterprise/auth/containers/SamlAuthCard/SamlAuthCard.tsx b/enterprise/frontend/src/metabase-enterprise/auth/containers/SamlAuthCard/SamlAuthCard.tsx new file mode 100644 index 000000000000..8f316762a167 --- /dev/null +++ b/enterprise/frontend/src/metabase-enterprise/auth/containers/SamlAuthCard/SamlAuthCard.tsx @@ -0,0 +1,20 @@ +import { t } from "ttag"; +import { connect } from "react-redux"; +import { getSetting } from "metabase/selectors/settings"; +import { updateSettings } from "metabase/admin/settings/settings"; +import AuthCard from "metabase/admin/settings/auth/components/AuthCard"; +import { Dispatch, State } from "metabase-types/store"; +import { SAML_SCHEMA } from "../../constants"; + +const mapStateToProps = (state: State) => ({ + type: "saml", + name: t`SAML`, + description: t`Allows users to login via a SAML Identity Provider.`, + isConfigured: getSetting(state, "saml-configured"), +}); + +const mapDispatchToProps = (dispatch: Dispatch) => ({ + onDeactivate: () => dispatch(updateSettings(SAML_SCHEMA.getDefault())), +}); + +export default connect(mapStateToProps, mapDispatchToProps)(AuthCard); diff --git a/enterprise/frontend/src/metabase-enterprise/auth/containers/SamlAuthCard/index.ts b/enterprise/frontend/src/metabase-enterprise/auth/containers/SamlAuthCard/index.ts new file mode 100644 index 000000000000..e3dc8d30e14d --- /dev/null +++ b/enterprise/frontend/src/metabase-enterprise/auth/containers/SamlAuthCard/index.ts @@ -0,0 +1 @@ +export { default } from "./SamlAuthCard"; diff --git a/enterprise/frontend/src/metabase-enterprise/auth/index.js b/enterprise/frontend/src/metabase-enterprise/auth/index.js index 13dd275fd198..7bce7a15dcbb 100644 --- a/enterprise/frontend/src/metabase-enterprise/auth/index.js +++ b/enterprise/frontend/src/metabase-enterprise/auth/index.js @@ -1,9 +1,6 @@ -import React from "react"; -import { t, jt } from "ttag"; +import { t } from "ttag"; import { updateIn } from "icepick"; -import ExternalLink from "metabase/core/components/ExternalLink"; import { LOGIN, LOGIN_GOOGLE } from "metabase/auth/actions"; - import { hasPremiumFeature } from "metabase-enterprise/settings"; import MetabaseSettings from "metabase/lib/settings"; import { @@ -13,8 +10,6 @@ import { PLUGIN_REDUX_MIDDLEWARES, } from "metabase/plugins"; -import AuthenticationOption from "metabase/admin/settings/components/widgets/AuthenticationOption"; -import AuthenticationWidget from "metabase/admin/settings/components/widgets/AuthenticationWidget"; import GroupMappingsWidget from "metabase/admin/settings/components/widgets/GroupMappingsWidget"; import SecretKeyWidget from "metabase/admin/settings/components/widgets/SecretKeyWidget"; import SessionTimeoutSetting from "metabase-enterprise/auth/components/SessionTimeoutSetting"; @@ -23,30 +18,25 @@ import SettingsGoogleForm from "metabase/admin/settings/components/SettingsGoogl import { createSessionMiddleware } from "../auth/middleware/session-middleware"; import SettingsSAMLForm from "./components/SettingsSAMLForm"; import SettingsJWTForm from "./components/SettingsJWTForm"; - import SSOButton from "./containers/SSOButton"; +import JwtAuthCard from "./containers/JwtAuthCard"; +import SamlAuthCard from "./containers/SamlAuthCard"; PLUGIN_ADMIN_SETTINGS_UPDATES.push(sections => updateIn(sections, ["authentication", "settings"], settings => [ ...settings, { - authName: t`SAML`, - authDescription: t`Allows users to login via a SAML Identity Provider.`, - authType: "saml", - authEnabled: settings => settings["saml-enabled"], - widget: AuthenticationOption, + key: "saml-enabled", + description: null, + noHeader: true, + widget: SamlAuthCard, getHidden: () => !hasPremiumFeature("sso"), }, { key: "jwt-enabled", description: null, - widget: AuthenticationWidget, - getProps: (setting, settings) => ({ - authName: t`JWT`, - authDescription: t`Allows users to login via a JWT Identity Provider.`, - authType: "jwt", - authConfigured: settings["jwt-configured"], - }), + noHeader: true, + widget: JwtAuthCard, getHidden: () => !hasPremiumFeature("sso"), }, { @@ -87,17 +77,7 @@ PLUGIN_ADMIN_SETTINGS_UPDATES.push(sections => ({ settings: [ { key: "saml-enabled", - display_name: t`SAML Authentication`, - description: jt`Use the settings below to configure your SSO via SAML. If you have any questions, check out our ${( - - {t`documentation`} - - )}.`, - type: "boolean", + getHidden: () => true, }, { key: "saml-identity-provider-uri", diff --git a/enterprise/frontend/src/metabase-enterprise/embedding/index.js b/enterprise/frontend/src/metabase-enterprise/embedding/index.js index 38571c3710d0..a7000674aed7 100644 --- a/enterprise/frontend/src/metabase-enterprise/embedding/index.js +++ b/enterprise/frontend/src/metabase-enterprise/embedding/index.js @@ -17,7 +17,11 @@ const APP_ORIGIN_SETTING = { description: ( <> {jt`With this Pro/Enterprise feature you can embed the full Metabase app. Enable your users to drill-through to charts, browse collections, and use the graphical query builder. ${( - + {t`Learn more.`} )}`} diff --git a/enterprise/frontend/src/metabase-enterprise/tools/mode.js b/enterprise/frontend/src/metabase-enterprise/tools/mode.js index f59162335c32..3c2dcff091e7 100644 --- a/enterprise/frontend/src/metabase-enterprise/tools/mode.js +++ b/enterprise/frontend/src/metabase-enterprise/tools/mode.js @@ -24,5 +24,5 @@ const ErrorDrill = ({ clicked }) => { export const ErrorMode = { name: "error", - drills: () => [ErrorDrill], + drills: [ErrorDrill], }; diff --git a/frontend/src/metabase-lib/Dimension.ts b/frontend/src/metabase-lib/Dimension.ts index bd1e24bd66a7..d49fda386579 100644 --- a/frontend/src/metabase-lib/Dimension.ts +++ b/frontend/src/metabase-lib/Dimension.ts @@ -410,6 +410,10 @@ export default class Dimension { return this._query; } + setQuery(query: StructuredQuery): Dimension { + return this; + } + sourceDimension() { return this._query && this._query.dimensionForSourceQuery(this); } @@ -731,6 +735,20 @@ export class FieldDimension extends Dimension { Object.freeze(this); } + setQuery(query: StructuredQuery): FieldDimension { + return new FieldDimension( + this._fieldIdOrName, + this._options, + this._metadata, + query, + { + _fieldInstance: this._fieldInstance, + _subDisplayName: this._subDisplayName, + _subTriggerDisplayName: this._subTriggerDisplayName, + }, + ); + } + isEqual(somethingElse) { if (isFieldDimension(somethingElse)) { return ( @@ -1154,6 +1172,15 @@ export class ExpressionDimension extends Dimension { Object.freeze(this); } + setQuery(query: StructuredQuery): ExpressionDimension { + return new ExpressionDimension( + this._expressionName, + this._options, + this._metadata, + query, + ); + } + isEqual(somethingElse) { if (isExpressionDimension(somethingElse)) { return ( @@ -1237,6 +1264,10 @@ export class ExpressionDimension extends Dimension { base_type = "type/Boolean"; break; + case MONOTYPE.DateTime: + base_type = "type/DateTime"; + break; + // fallback default: base_type = "type/Float"; @@ -1414,6 +1445,15 @@ export class AggregationDimension extends Dimension { Object.freeze(this); } + setQuery(query: StructuredQuery): AggregationDimension { + return new AggregationDimension( + this._aggregationIndex, + this._options, + this._metadata, + query, + ); + } + aggregationIndex(): number { return this._aggregationIndex; } diff --git a/frontend/src/metabase-lib/Mode/Mode.ts b/frontend/src/metabase-lib/Mode/Mode.ts index 4c29dbcd8e0b..dc1a5fcb23b3 100644 --- a/frontend/src/metabase-lib/Mode/Mode.ts +++ b/frontend/src/metabase-lib/Mode/Mode.ts @@ -27,13 +27,15 @@ export default class Mode { settings: Record, extraData: Record, ): ClickAction[] { - return this._queryMode.drills().flatMap(actionCreator => - actionCreator({ - question: this._question, - settings, - clicked, - extraData, - }), - ); + const mode = this._queryMode; + const question = this._question; + const props = { question, settings, clicked, extraData }; + const actions = mode.drills.flatMap(drill => drill(props)); + + if (!actions.length && mode.fallback) { + return mode.fallback(props); + } else { + return actions; + } } } diff --git a/frontend/src/metabase-lib/Question.ts b/frontend/src/metabase-lib/Question.ts index ff183471ca5f..fcac701b5bad 100644 --- a/frontend/src/metabase-lib/Question.ts +++ b/frontend/src/metabase-lib/Question.ts @@ -194,7 +194,7 @@ class QuestionInner { } } - throw new Error("Unknown query type: " + datasetQuery.type); + console.warn("Unknown query type: " + datasetQuery?.type); } isNative(): boolean { @@ -272,7 +272,7 @@ class QuestionInner { } isAction() { - return this._card && this._card.is_write; + return false; } setPersisted(isPersisted) { @@ -290,7 +290,7 @@ class QuestionInner { } setIsAction(isAction) { - return this.setCard(assoc(this.card(), "is_write", isAction)); + return this.card(); } // locking the display prevents auto-selection diff --git a/frontend/src/metabase-lib/expressions/config.js b/frontend/src/metabase-lib/expressions/config.js index 9d07b368aef0..1d6cab352f23 100644 --- a/frontend/src/metabase-lib/expressions/config.js +++ b/frontend/src/metabase-lib/expressions/config.js @@ -324,58 +324,71 @@ export const MBQL_CLAUSES = { "get-year": { displayName: `year`, type: "number", - args: ["expression"], + args: ["datetime"], }, "get-quarter": { displayName: `quarter`, type: "number", - args: ["expression"], + args: ["datetime"], }, "get-month": { displayName: `month`, type: "number", - args: ["expression"], + args: ["datetime"], }, "get-week": { displayName: `week`, type: "number", - args: ["expression"], + args: ["datetime"], hasOptions: true, // optional mode parameter }, "get-day": { displayName: `day`, type: "number", - args: ["expression"], + args: ["datetime"], }, "get-day-of-week": { displayName: `weekday`, type: "number", - args: ["expression"], + args: ["datetime"], }, "get-hour": { displayName: `hour`, type: "number", - args: ["expression"], + args: ["datetime"], }, "get-minute": { displayName: `minute`, type: "number", - args: ["expression"], + args: ["datetime"], }, "get-second": { displayName: `second`, type: "number", - args: ["expression"], + args: ["datetime"], + }, + "datetime-diff": { + displayName: `datetimeDiff`, + type: "number", + args: ["datetime", "datetime", "string"], + requiresFeature: "datetime-diff", }, "datetime-add": { displayName: `datetimeAdd`, - type: "expression", - args: ["expression", "number", "string"], + type: "datetime", + args: ["datetime", "number", "string"], }, "datetime-subtract": { displayName: `datetimeSubtract`, - type: "expression", - args: ["expression", "number", "string"], + type: "datetime", + args: ["datetime", "number", "string"], + }, + "convert-timezone": { + displayName: `convertTimezone`, + type: "datetime", + args: ["datetime", "string"], + hasOptions: true, + requiresFeature: "convert-timezone", }, }; @@ -448,6 +461,7 @@ export const EXPRESSION_FUNCTIONS = new Set([ "power", "log", "exp", + "datetime-diff", // date/time "get-year", "get-quarter", @@ -460,6 +474,7 @@ export const EXPRESSION_FUNCTIONS = new Set([ "get-second", "datetime-add", "datetime-subtract", + "convert-timezone", // boolean "contains", "ends-with", diff --git a/frontend/src/metabase-lib/expressions/helper-text-strings.ts b/frontend/src/metabase-lib/expressions/helper-text-strings.ts index eeb080d7f1cf..620827a250cb 100644 --- a/frontend/src/metabase-lib/expressions/helper-text-strings.ts +++ b/frontend/src/metabase-lib/expressions/helper-text-strings.ts @@ -381,6 +381,37 @@ const helperTextStrings: HelpText[] = [ }, ], }, + { + name: "datetime-diff", + structure: + "datetimeDiff(" + + t`datetime1` + + ", " + + t`datetime2` + + ", " + + t`unit` + + ")", + description: t`Get the difference between two datetime values (datetime2 minus datetime1) using the specified unit of time.`, + example: + "datetimeDiff([" + + t`created_at` + + "], [" + + t`shipped_at` + + "], " + + t`"month"` + + ")", + args: [ + { + name: t`datetime1, datetime2`, + description: t`The columns or expressions with your datetime values.`, + }, + { + name: t`unit`, + description: t`Choose from: "year", "month", "week", "day", "hour", "minute", or "second".`, + }, + ], + docsPage: "datetimediff", + }, { name: "exp", structure: "exp(" + t`column` + ")", @@ -611,12 +642,12 @@ const helperTextStrings: HelpText[] = [ { name: "get-week", structure: "week(" + t`column` + ", " + t`mode` + ")", - description: t`Extracts the week of the year as an integer..`, + description: t`Extracts the week of the year as an integer.`, example: "week([" + t`Created At` + '], "iso")', args: [ { name: t`column`, - description: t`The name of the column with your date or datetime value..`, + description: t`The name of the column with your date or datetime value.`, }, { name: t`mode`, @@ -708,6 +739,7 @@ const helperTextStrings: HelpText[] = [ description: t`"year", "month", "quarter", "day", "hour", "minute", "second" or "millisecond".`, }, ], + docsPage: "datetimeadd", }, { name: "datetime-subtract", @@ -729,6 +761,38 @@ const helperTextStrings: HelpText[] = [ description: t`"year", "month", "quarter", "day", "hour", "minute", "second" or "millisecond".`, }, ], + docsPage: "datetimesubtract", + }, + { + name: "convert-timezone", + structure: + "convertTimezone(" + + t`column` + + ", " + + t`target` + + ", [" + + t`source` + + "])", + description: t`Convert timezone of a date or timestamp column. +We support tz database time zone names. +See the full list here: https://w.wiki/4Jx`, + example: + "convertTimezone([" + t`Created At` + '], "Asia/Ho_Chi_Minh", "UTC")', + args: [ + { + name: t`column`, + description: t`The column with your date or timestamp values.`, + }, + { + name: t`target`, + description: t`The timezone you want to assign to your column.`, + }, + { + name: t`source`, + description: t`The current time zone. Only required for timestamps with no time zone.`, + }, + ], + docsPage: "converttimezone", }, ]; diff --git a/frontend/src/metabase-lib/expressions/resolver.js b/frontend/src/metabase-lib/expressions/resolver.js index 52c9ac8d7804..2612db2c76d0 100644 --- a/frontend/src/metabase-lib/expressions/resolver.js +++ b/frontend/src/metabase-lib/expressions/resolver.js @@ -38,11 +38,24 @@ function findMBQL(op) { return clause; } +// a is the type of the argument expected, +// as defined in MBQL_CLAUSES, +// and b is the inferred type of the argument const isCompatible = (a, b) => { if (a === b) { return true; } - if (a === "expression" && (b === "number" || b === "string")) { + + // if b is a string, then it can be an arg to a function that expects a datetime argument. + // This allows datetime string literals to work as args for functions that expect datetime types. + // FIXME: By doing this we are allowing string columns to be arguments to functions, which isn’t valid MBQL. + if (a === "datetime" && b === "string") { + return true; + } + if ( + a === "expression" && + (b === "datetime" || b === "number" || b === "string") + ) { return true; } if (a === "aggregation" && b === "number") { diff --git a/frontend/src/metabase-lib/expressions/typeinferencer.js b/frontend/src/metabase-lib/expressions/typeinferencer.js index 3dc1d5c1fdc0..5c0cb61caf58 100644 --- a/frontend/src/metabase-lib/expressions/typeinferencer.js +++ b/frontend/src/metabase-lib/expressions/typeinferencer.js @@ -6,6 +6,7 @@ export const MONOTYPE = { Number: "number", String: "string", Boolean: "boolean", + DateTime: "datetime", }; export function infer(mbql, env) { @@ -37,8 +38,6 @@ export function infer(mbql, env) { case "case": return infer(mbql[1][0][1], env); case "coalesce": - case "date-add": - case "date-subtract": return infer(mbql[1], env); } @@ -46,6 +45,8 @@ export function infer(mbql, env) { if (func) { const returnType = func.type; switch (returnType) { + case "datetime": + return MONOTYPE.DateTime; case "object": return MONOTYPE.Undefined; case "aggregation": diff --git a/frontend/src/metabase-lib/metadata/Database.ts b/frontend/src/metabase-lib/metadata/Database.ts index 0bffa63e426d..1b9ce2113d67 100644 --- a/frontend/src/metabase-lib/metadata/Database.ts +++ b/frontend/src/metabase-lib/metadata/Database.ts @@ -20,6 +20,7 @@ import Metadata from "./Metadata"; class DatabaseInner extends Base { id: number; name: string; + engine: string; description: string; tables: Table[]; schemas: Schema[]; diff --git a/frontend/src/metabase-lib/parameters/utils/mbql.js b/frontend/src/metabase-lib/parameters/utils/mbql.js index e92532398bb7..9f603386b234 100644 --- a/frontend/src/metabase-lib/parameters/utils/mbql.js +++ b/frontend/src/metabase-lib/parameters/utils/mbql.js @@ -13,7 +13,10 @@ import { isDateParameter, } from "metabase-lib/parameters/utils/parameter-type"; import { isTemplateTagReference } from "metabase-lib/references"; -import { getParameterOperatorName } from "metabase-lib/parameters/utils/operators"; +import { + deriveFieldOperatorFromParameter, + getParameterOperatorName, +} from "metabase-lib/parameters/utils/operators"; import { hasParameterValue } from "metabase-lib/parameters/utils/parameter-values"; const withTemporalUnit = (fieldRef, unit) => { @@ -145,10 +148,13 @@ export function dateParameterValueToMBQL(parameterValue, fieldRef) { export function stringParameterValueToMBQL(parameter, fieldRef) { const parameterValue = parameter.value; + const operator = deriveFieldOperatorFromParameter(parameter); const subtype = getParameterSubType(parameter); const operatorName = getParameterOperatorName(subtype); - return [operatorName, fieldRef].concat(parameterValue); + return [operatorName, fieldRef] + .concat(parameterValue) + .concat(operator?.optionsDefaults ?? []); } export function numberParameterValueToMBQL(parameter, fieldRef) { diff --git a/frontend/src/metabase-lib/parameters/utils/mbql.unit.spec.js b/frontend/src/metabase-lib/parameters/utils/mbql.unit.spec.js index 84b6c577f2a7..4825f6b93c6b 100644 --- a/frontend/src/metabase-lib/parameters/utils/mbql.unit.spec.js +++ b/frontend/src/metabase-lib/parameters/utils/mbql.unit.spec.js @@ -183,7 +183,14 @@ describe("parameters/utils/mbql", () => { { type: "string/starts-with", value: "1" }, null, ), - ).toEqual(["starts-with", null, "1"]); + ).toEqual([ + "starts-with", + null, + "1", + { + "case-sensitive": false, + }, + ]); }); }); @@ -278,12 +285,19 @@ describe("parameters/utils/mbql", () => { fieldFilterParameterToMBQLFilter( { target: ["dimension", ["field", PRODUCTS.CATEGORY.id, null]], - type: "string/starts-with", + type: "string/contains", value: "foo", }, metadata, ), - ).toEqual(["starts-with", ["field", PRODUCTS.CATEGORY.id, null], "foo"]); + ).toEqual([ + "contains", + ["field", PRODUCTS.CATEGORY.id, null], + "foo", + { + "case-sensitive": false, + }, + ]); expect( fieldFilterParameterToMBQLFilter( @@ -294,7 +308,12 @@ describe("parameters/utils/mbql", () => { }, metadata, ), - ).toEqual(["starts-with", ["field", PRODUCTS.CATEGORY.id, null], "foo"]); + ).toEqual([ + "starts-with", + ["field", PRODUCTS.CATEGORY.id, null], + "foo", + { "case-sensitive": false }, + ]); }); it("should return mbql filter for category parameter", () => { diff --git a/frontend/src/metabase-lib/parameters/utils/parameter-type.ts b/frontend/src/metabase-lib/parameters/utils/parameter-type.ts index d914270c68c1..3dd465a5a261 100644 --- a/frontend/src/metabase-lib/parameters/utils/parameter-type.ts +++ b/frontend/src/metabase-lib/parameters/utils/parameter-type.ts @@ -22,6 +22,11 @@ function splitType(parameterOrType: Parameter | string) { return parameterType.split("/"); } +export function isIdParameter(parameter: Parameter | string) { + const type = getParameterType(parameter); + return type === "id"; +} + export function isDateParameter(parameter: Parameter | string) { const type = getParameterType(parameter); return type === "date"; diff --git a/frontend/src/metabase-lib/queries/InternalQuery.ts b/frontend/src/metabase-lib/queries/InternalQuery.ts index 26a6ae71bba0..f2c2928680ae 100644 --- a/frontend/src/metabase-lib/queries/InternalQuery.ts +++ b/frontend/src/metabase-lib/queries/InternalQuery.ts @@ -11,6 +11,6 @@ import AtomicQuery from "metabase-lib/queries/AtomicQuery"; // Internal queries export default class InternalQuery extends AtomicQuery { static isDatasetQueryType(datasetQuery: DatasetQuery) { - return datasetQuery.type === "internal"; + return datasetQuery?.type === "internal"; } } diff --git a/frontend/src/metabase-lib/queries/NativeQuery.ts b/frontend/src/metabase-lib/queries/NativeQuery.ts index fbb90ea01582..124134fe6891 100644 --- a/frontend/src/metabase-lib/queries/NativeQuery.ts +++ b/frontend/src/metabase-lib/queries/NativeQuery.ts @@ -137,7 +137,7 @@ export default class NativeQuery extends AtomicQuery { } static isDatasetQueryType(datasetQuery: DatasetQuery) { - return datasetQuery && datasetQuery.type === NATIVE_QUERY_TEMPLATE.type; + return datasetQuery?.type === NATIVE_QUERY_TEMPLATE.type; } /* Query superclass methods */ diff --git a/frontend/src/metabase-lib/queries/StructuredQuery.ts b/frontend/src/metabase-lib/queries/StructuredQuery.ts index bb4b97b7e765..f11c638f80c3 100644 --- a/frontend/src/metabase-lib/queries/StructuredQuery.ts +++ b/frontend/src/metabase-lib/queries/StructuredQuery.ts @@ -99,7 +99,7 @@ export interface SegmentOption { class StructuredQueryInner extends AtomicQuery { static isDatasetQueryType(datasetQuery: DatasetQuery) { - return datasetQuery && datasetQuery.type === STRUCTURED_QUERY_TEMPLATE.type; + return datasetQuery?.type === STRUCTURED_QUERY_TEMPLATE.type; } // For Flow type completion diff --git a/frontend/src/metabase-lib/queries/drills/native-drill-fallback.ts b/frontend/src/metabase-lib/queries/drills/native-drill-fallback.ts new file mode 100644 index 000000000000..9fce2f62fe28 --- /dev/null +++ b/frontend/src/metabase-lib/queries/drills/native-drill-fallback.ts @@ -0,0 +1,17 @@ +import Question from "metabase-lib/Question"; + +interface FallbackNativeDrillProps { + question: Question; +} + +export function nativeDrillFallback({ question }: FallbackNativeDrillProps) { + const query = question.query(); + const database = question.database(); + if (!question.isNative() || !query.isEditable() || !database) { + return null; + } + + return { + database, + }; +} diff --git a/frontend/src/metabase-lib/queries/structured/Join.ts b/frontend/src/metabase-lib/queries/structured/Join.ts index a87ecefe92e8..ef97f69eb74f 100644 --- a/frontend/src/metabase-lib/queries/structured/Join.ts +++ b/frontend/src/metabase-lib/queries/structured/Join.ts @@ -625,7 +625,7 @@ export default class Join extends MBQLObjectClause { joinedDimension(dimension: Dimension) { if (dimension instanceof FieldDimension) { - return dimension.withJoinAlias(this.alias); + return dimension.withJoinAlias(this.alias).setQuery(this.query()); } console.warn("Don't know how to create joined dimension with:", dimension); diff --git a/frontend/src/metabase-lib/queries/utils/card.js b/frontend/src/metabase-lib/queries/utils/card.js index b3d75c9a6c77..a528e13d6e40 100644 --- a/frontend/src/metabase-lib/queries/utils/card.js +++ b/frontend/src/metabase-lib/queries/utils/card.js @@ -117,24 +117,24 @@ export function applyParameters( const options = deriveFieldOperatorFromParameter(parameter)?.optionsDefaults; + const queryParameter = { + type, + value: normalizeParameterValue(type, value), + id: parameter.id, + }; + + if (options) { + queryParameter.options = options; + } + if (mapping) { // mapped target, e.x. on a dashboard - datasetQuery.parameters.push({ - type, - value: normalizeParameterValue(type, value), - target: mapping.target, - options, - id: parameter.id, - }); + queryParameter.target = mapping.target; + datasetQuery.parameters.push(queryParameter); } else if (parameter.target) { // inline target, e.x. on a card - datasetQuery.parameters.push({ - type, - value: normalizeParameterValue(type, value), - target: parameter.target, - options, - id: parameter.id, - }); + queryParameter.target = parameter.target; + datasetQuery.parameters.push(queryParameter); } } diff --git a/frontend/src/metabase-lib/queries/utils/get-column-key.ts b/frontend/src/metabase-lib/queries/utils/get-column-key.ts index 7e3505085363..38f8f752b244 100644 --- a/frontend/src/metabase-lib/queries/utils/get-column-key.ts +++ b/frontend/src/metabase-lib/queries/utils/get-column-key.ts @@ -17,7 +17,6 @@ export const getColumnKey = ( let fieldRef = column.field_ref; if (!fieldRef) { - console.error("column is missing field_ref", column); fieldRef = createFieldReference(column.name); } diff --git a/frontend/src/metabase-lib/queries/utils/structured-query-table.ts b/frontend/src/metabase-lib/queries/utils/structured-query-table.ts index eb2ab083efc0..39534e75e44e 100644 --- a/frontend/src/metabase-lib/queries/utils/structured-query-table.ts +++ b/frontend/src/metabase-lib/queries/utils/structured-query-table.ts @@ -50,6 +50,7 @@ function getFieldsForSourceQueryTable( const virtualField = createVirtualField({ ...column, id, + source: "fields", query: originalQuery, metadata, }); diff --git a/frontend/src/metabase-types/api/card.ts b/frontend/src/metabase-types/api/card.ts index ea0b5d6eeadb..59ba6670d529 100644 --- a/frontend/src/metabase-types/api/card.ts +++ b/frontend/src/metabase-types/api/card.ts @@ -36,8 +36,9 @@ export type SeriesSettings = { export type SeriesOrderSetting = { name: string; - originalIndex: number; + key: string; enabled: boolean; + color?: string; }; export type VisualizationSettings = { @@ -68,6 +69,9 @@ export type VisualizationSettings = { "graph.series_order"?: SeriesOrderSetting[]; + // Funnel settings + "funnel.rows"?: SeriesOrderSetting[]; + [key: string]: any; }; diff --git a/frontend/src/metabase-types/api/dataset.ts b/frontend/src/metabase-types/api/dataset.ts index c9f9d00aeaf7..210a18cb54e9 100644 --- a/frontend/src/metabase-types/api/dataset.ts +++ b/frontend/src/metabase-types/api/dataset.ts @@ -2,6 +2,7 @@ import type { DatetimeUnit, DimensionReference, } from "metabase-types/api/query"; +import { Card } from "./card"; import { DatabaseId } from "./database"; import { DownloadPermission } from "./permissions"; @@ -13,12 +14,15 @@ export interface DatasetColumn { display_name: string; source: string; name: string; + // FIXME: this prop does not come from API remapped_to_column?: DatasetColumn; unit?: DatetimeUnit; field_ref?: DimensionReference; expression_name?: any; base_type?: string; semantic_type?: string; + remapped_from?: string; + remapped_to?: string; binning_info?: { bin_width?: number; }; @@ -37,3 +41,17 @@ export interface Dataset { row_count: number; running_time: number; } + +export type SingleSeries = { + card: Card; + data: DatasetData; + error_type?: string; + error?: { + status: number; // HTTP status code + data?: string; + }; +}; + +export type RawSeries = SingleSeries[]; +export type TransformedSeries = RawSeries & { _raw: Series }; +export type Series = RawSeries | TransformedSeries; diff --git a/frontend/src/metabase-types/api/mocks/card.ts b/frontend/src/metabase-types/api/mocks/card.ts index 9e99c6927f58..33921c1323e9 100644 --- a/frontend/src/metabase-types/api/mocks/card.ts +++ b/frontend/src/metabase-types/api/mocks/card.ts @@ -3,6 +3,7 @@ import { Card, UnsavedCard, VisualizationSettings, + SeriesOrderSetting, } from "metabase-types/api"; import { createMockStructuredDatasetQuery } from "./query"; @@ -37,6 +38,18 @@ export const createMockVisualizationSettings = ( ...opts, }); +export const createMockSeriesOrderSetting = ({ + name = "", + key, + enabled = true, + ...opts +}: Partial): SeriesOrderSetting => ({ + name, + key: key || name, + enabled, + ...opts, +}); + export const createMockModerationReview = ( opts?: Partial, ): ModerationReview => ({ diff --git a/frontend/src/metabase-types/api/mocks/dataset.ts b/frontend/src/metabase-types/api/mocks/dataset.ts index 2617738ceb38..11c4542db410 100644 --- a/frontend/src/metabase-types/api/mocks/dataset.ts +++ b/frontend/src/metabase-types/api/mocks/dataset.ts @@ -13,7 +13,7 @@ export const createMockColumn = (data: Partial) => { }; }; -type MockDatasetOpts = Partial> & { +export type MockDatasetOpts = Partial> & { data?: Partial; }; diff --git a/frontend/src/metabase-types/api/mocks/series.ts b/frontend/src/metabase-types/api/mocks/series.ts new file mode 100644 index 000000000000..d049fc542498 --- /dev/null +++ b/frontend/src/metabase-types/api/mocks/series.ts @@ -0,0 +1,18 @@ +import { SingleSeries, Series } from "../dataset"; +import { Card } from "../card"; +import { createMockCard } from "./card"; +import { createMockDataset, MockDatasetOpts } from "./dataset"; + +export const createMockSingleSeries = ( + cardOpts: Partial, + dataOpts: MockDatasetOpts = {}, +): SingleSeries => { + return { + card: createMockCard(cardOpts), + data: createMockDataset(dataOpts).data, + }; +}; + +export const createMockSeries = (opts: { name: string }[]): Series => { + return opts.map(opt => createMockSingleSeries({ name: opt.name })); +}; diff --git a/frontend/src/metabase-types/api/mocks/settings.ts b/frontend/src/metabase-types/api/mocks/settings.ts index b14d44d2bb42..8e58d7c5a17d 100644 --- a/frontend/src/metabase-types/api/mocks/settings.ts +++ b/frontend/src/metabase-types/api/mocks/settings.ts @@ -62,29 +62,38 @@ export const createMockSettings = (opts?: Partial): Settings => ({ "application-font": "Lato", "application-font-files": [], "available-fonts": [], - "available-locales": [], + "available-locales": null, + "custom-formatting": {}, + "deprecation-notice-version": undefined, + "email-configured?": false, "enable-public-sharing": false, "enable-xrays": false, - "email-configured?": false, - engines: createMockEngines(), - "is-hosted?": false, + "google-auth-auto-create-accounts-domain": null, "google-auth-client-id": null, + "google-auth-configured": false, + "google-auth-enabled": false, + "is-hosted?": false, + "jwt-enabled": false, + "jwt-configured": false, + "ldap-configured?": false, "ldap-enabled": false, "loading-message": "doing-science", - "deprecation-notice-version": undefined, + "saml-configured": false, + "saml-enabled": false, "session-cookies": null, - "site-locale": "en", "show-database-syncing-modal": false, "show-homepage-data": false, - "show-homepage-xrays": false, "show-homepage-pin-message": false, + "show-homepage-xrays": false, "show-lighthouse-illustration": true, "show-metabot": true, - "slack-token": undefined, - "token-status": createMockTokenStatus(), + "site-locale": "en", + "slack-app-token": null, + "slack-files-channel": null, + "slack-token": null, "slack-token-valid?": false, - "slack-app-token": undefined, - "slack-files-channel": undefined, + "token-status": createMockTokenStatus(), + engines: createMockEngines(), version: createMockVersion(), ...opts, }); diff --git a/frontend/src/metabase-types/api/mocks/slack.ts b/frontend/src/metabase-types/api/mocks/slack.ts index b23271ce86e3..c7a905a0dab9 100644 --- a/frontend/src/metabase-types/api/mocks/slack.ts +++ b/frontend/src/metabase-types/api/mocks/slack.ts @@ -3,5 +3,7 @@ import { SlackSettings } from "metabase-types/api"; export const createMockSlackSettings = ( opts?: Partial, ): SlackSettings => ({ + "slack-app-token": null, + "slack-files-channel": null, ...opts, }); diff --git a/frontend/src/metabase-types/api/mocks/timeline.ts b/frontend/src/metabase-types/api/mocks/timeline.ts index b0ed0639bba6..6c572d8fa884 100644 --- a/frontend/src/metabase-types/api/mocks/timeline.ts +++ b/frontend/src/metabase-types/api/mocks/timeline.ts @@ -1,11 +1,23 @@ -import { TimelineEvent, Timeline } from "../timeline"; +import { TimelineEvent, Timeline, TimelineData } from "../timeline"; import { createMockUser } from "./user"; export const createMockTimeline = (opts?: Partial): Timeline => ({ id: 1, collection_id: 1, name: "Events", - description: "A timeline of events", + description: null, + icon: "star", + default: false, + archived: false, + ...opts, +}); + +export const createMockTimelineData = ( + opts?: Partial, +): TimelineData => ({ + collection_id: 1, + name: "Events", + description: null, icon: "star", default: false, archived: false, diff --git a/frontend/src/metabase-types/api/settings.ts b/frontend/src/metabase-types/api/settings.ts index 0f639288d219..d717dabb6d67 100644 --- a/frontend/src/metabase-types/api/settings.ts +++ b/frontend/src/metabase-types/api/settings.ts @@ -1,8 +1,32 @@ +export interface FormattingSettings { + "type/Temporal"?: DateFormattingSettings; + "type/Number"?: NumberFormattingSettings; + "type/Currency"?: CurrencyFormattingSettings; +} + +export interface DateFormattingSettings { + date_style?: string; + date_separator?: string; + date_abbreviate?: boolean; + time_style?: string; +} + +export interface NumberFormattingSettings { + number_separators?: string; +} + +export interface CurrencyFormattingSettings { + currency?: string; + currency_style?: string; + currency_in_header?: boolean; +} + export interface Engine { "driver-name": string; "superseded-by": string | undefined; source: EngineSource; } + export interface EngineSource { type?: "official" | "community" | "partner"; contact?: EngineSourceContact; @@ -42,28 +66,37 @@ export interface Settings { "application-font": string; "application-font-files": FontFile[] | null; "available-fonts": string[]; - "available-locales": LocaleData[] | undefined; + "available-locales": LocaleData[] | null; + "custom-formatting": FormattingSettings; + "deprecation-notice-version": string | undefined; + "email-configured?": boolean; "enable-public-sharing": boolean; "enable-xrays": boolean; - "email-configured?": boolean; - engines: Record; - "is-hosted?": boolean; + "google-auth-auto-create-accounts-domain": string | null; "google-auth-client-id": string | null; - "deprecation-notice-version": string | undefined; + "google-auth-configured": boolean; + "google-auth-enabled": boolean; + "is-hosted?": boolean; + "jwt-enabled"?: boolean; + "jwt-configured"?: boolean; + "ldap-configured?": boolean; "ldap-enabled": boolean; "loading-message": LoadingMessage; + "saml-configured"?: boolean; + "saml-enabled"?: boolean; "session-cookies": boolean | null; - "site-locale": string; "show-database-syncing-modal": boolean; "show-homepage-data": boolean; - "show-homepage-xrays": boolean; "show-homepage-pin-message": boolean; + "show-homepage-xrays": boolean; "show-lighthouse-illustration": boolean; "show-metabot": boolean; - "slack-token": string | undefined; + "site-locale": string; + "slack-app-token": string | null; + "slack-files-channel": string | null; + "slack-token": string | null; "slack-token-valid?": boolean; - "slack-app-token": string | undefined; - "slack-files-channel": string | undefined; "token-status": TokenStatus | undefined; + engines: Record; version: Version; } diff --git a/frontend/src/metabase-types/api/slack.ts b/frontend/src/metabase-types/api/slack.ts index 96659065f39e..9719863fe99b 100644 --- a/frontend/src/metabase-types/api/slack.ts +++ b/frontend/src/metabase-types/api/slack.ts @@ -1,4 +1,4 @@ export interface SlackSettings { - "slack-app-token"?: string; - "slack-files-channel"?: string; + "slack-app-token": string | null; + "slack-files-channel": string | null; } diff --git a/frontend/src/metabase-types/api/timeline.ts b/frontend/src/metabase-types/api/timeline.ts index c2db3554b0df..9deeee97a11d 100644 --- a/frontend/src/metabase-types/api/timeline.ts +++ b/frontend/src/metabase-types/api/timeline.ts @@ -1,9 +1,14 @@ -import { Collection } from "./collection"; +import { CardId } from "./card"; +import { Collection, RegularCollectionId } from "./collection"; import { User } from "./user"; +export type TimelineId = number; +export type TimelineEventId = number; +export type TimelineEventSource = "question" | "collections" | "api"; + export interface Timeline { - id: number; - collection_id: number | null; + id: TimelineId; + collection_id: RegularCollectionId | null; name: string; description: string | null; icon: string; @@ -13,9 +18,19 @@ export interface Timeline { events?: TimelineEvent[]; } +export interface TimelineData { + id?: number; + collection_id: RegularCollectionId | null; + name: string; + description: string | null; + icon: string; + default: boolean; + archived: boolean; +} + export interface TimelineEvent { id: number; - timeline_id: number; + timeline_id: TimelineId; name: string; description: string | null; icon: string; @@ -27,4 +42,16 @@ export interface TimelineEvent { created_at: string; } -export type TimelineEventSource = "question" | "collections" | "api"; +export interface TimelineEventData { + id?: number; + timeline_id?: TimelineId; + name: string; + description: string | null; + icon: string; + timestamp: string; + timezone: string; + time_matters: boolean; + archived: boolean; + source?: TimelineEventSource; + question_id?: CardId; +} diff --git a/frontend/src/metabase-types/forms/index.ts b/frontend/src/metabase-types/forms/index.ts index d26e30159702..974ef36da32e 100644 --- a/frontend/src/metabase-types/forms/index.ts +++ b/frontend/src/metabase-types/forms/index.ts @@ -1,8 +1,21 @@ +/** + * @deprecated + */ export type FieldName = string; + +/** + * @deprecated + */ export type DefaultFieldValue = unknown; +/** + * @deprecated + */ export type FieldValues = Record; +/** + * @deprecated + */ export type BaseFieldValues = { [field: string]: any; }; @@ -10,6 +23,9 @@ export type BaseFieldValues = { type FieldValidateResultOK = undefined | null | false; type FieldValidateResultError = string; +/** + * @deprecated + */ export type Validator = ( value: string, ) => FieldValidateResultOK | FieldValidateResultError; @@ -17,6 +33,9 @@ export type Validator = ( // Extending Record type here as field definition's props // will be just spread to the final field widget // (e.g. autoFocus, placeholder) +/** + * @deprecated + */ export type BaseFieldDefinition = Record & { name: string; type?: string; @@ -39,19 +58,31 @@ export type BaseFieldDefinition = Record & { normalize?: (value: any) => DefaultFieldValue; }; +/** + * @deprecated + */ export type StandardFormFieldDefinition = BaseFieldDefinition & { // If not is not provided, we're going to use default text input type?: string | (() => JSX.Element); }; +/** + * @deprecated + */ export type CustomFormFieldDefinition = BaseFieldDefinition & { widget: () => JSX.Element; }; +/** + * @deprecated + */ export type FormFieldDefinition = | StandardFormFieldDefinition | CustomFormFieldDefinition; +/** + * @deprecated + */ export type FormField = { name: keyof Values; value: Value; @@ -71,10 +102,16 @@ export type FormField = { onChange: (value: Value) => void; }; +/** + * @deprecated + */ export type FormObject = { fields: FormFieldDefinition[] | ((values?: Values) => FormFieldDefinition[]); }; +/** + * @deprecated + */ export type PopulatedFormObject = { fields: (values?: Values) => FormFieldDefinition[]; fieldNames: (values?: Partial) => (keyof Values)[]; diff --git a/frontend/src/metabase-types/forms/legacy.ts b/frontend/src/metabase-types/forms/legacy.ts index df21f36b3e87..c7e4b5fd44c3 100644 --- a/frontend/src/metabase-types/forms/legacy.ts +++ b/frontend/src/metabase-types/forms/legacy.ts @@ -1,6 +1,16 @@ +/** + * @deprecated + */ export type FieldName = string; + +/** + * @deprecated + */ export type DefaultFieldValue = unknown; +/** + * @deprecated + */ export type FieldValues = Record; type FieldValidateResultOK = undefined; @@ -9,6 +19,9 @@ type FieldValidateResultError = string; // Extending Record type here as field definition's props // will be just spread to the final field widget // (e.g. autoFocus, placeholder) +/** + * @deprecated + */ export type BaseFieldDefinition = Record & { name: string; type?: string; @@ -33,19 +46,31 @@ export type BaseFieldDefinition = Record & { normalize?: (value: unknown) => DefaultFieldValue; }; +/** + * @deprecated + */ export type StandardFormFieldDefinition = BaseFieldDefinition & { // If not is not provided, we're going to use default text input type?: string | (() => JSX.Element); }; +/** + * @deprecated + */ export type CustomFormFieldDefinition = BaseFieldDefinition & { widget: () => JSX.Element; }; +/** + * @deprecated + */ export type FormFieldDefinition = | StandardFormFieldDefinition | CustomFormFieldDefinition; +/** + * @deprecated + */ export type FormField = { name: FieldName; value: Value; @@ -64,12 +89,18 @@ export type FormField = { onFocus: () => void; }; +/** + * @deprecated + */ export type FormObject = { fields: | FormFieldDefinition[] | ((values?: FieldValues) => FormFieldDefinition[]); }; +/** + * @deprecated + */ export type PopulatedFormObject = { fields: (values?: FieldValues) => FormFieldDefinition[]; fieldNames: (values: FieldValues) => FieldName[]; diff --git a/frontend/src/metabase-types/store/embed.ts b/frontend/src/metabase-types/store/embed.ts index b45dca33c162..8627cd9f4cec 100644 --- a/frontend/src/metabase-types/store/embed.ts +++ b/frontend/src/metabase-types/store/embed.ts @@ -3,6 +3,7 @@ export interface EmbedOptions { top_nav?: boolean; search?: boolean; new_button?: boolean; + breadcrumbs?: boolean; side_nav?: boolean | "default"; header?: boolean; additional_info?: boolean; diff --git a/frontend/src/metabase-types/store/setup.ts b/frontend/src/metabase-types/store/setup.ts index be113f982a3a..e015cf665d84 100644 --- a/frontend/src/metabase-types/store/setup.ts +++ b/frontend/src/metabase-types/store/setup.ts @@ -4,8 +4,8 @@ export interface Locale { } export interface UserInfo { - first_name: string; - last_name: string; + first_name: string | null; + last_name: string | null; email: string; site_name: string; password: string; @@ -13,8 +13,8 @@ export interface UserInfo { } export interface InviteInfo { - first_name: string; - last_name: string; + first_name: string | null; + last_name: string | null; email: string; } diff --git a/frontend/src/metabase-types/types/Visualization.ts b/frontend/src/metabase-types/types/Visualization.ts index 5d7afe53dab5..799e7acb3ca9 100644 --- a/frontend/src/metabase-types/types/Visualization.ts +++ b/frontend/src/metabase-types/types/Visualization.ts @@ -15,7 +15,8 @@ export type ActionCreator = (props: ClickActionProps) => ClickAction[]; export type QueryMode = { name: string; - drills: () => ActionCreator[]; + drills: ActionCreator[]; + fallback?: ActionCreator; }; export type HoverData = Array<{ key: string; value: any; col?: Column }>; diff --git a/frontend/src/metabase/App.tsx b/frontend/src/metabase/App.tsx index 316cac6215de..c3437b3c4c4a 100644 --- a/frontend/src/metabase/App.tsx +++ b/frontend/src/metabase/App.tsx @@ -15,7 +15,7 @@ import { getErrorPage, getIsAdminApp, getIsAppBarVisible, - getIsNavBarVisible, + getIsNavBarEnabled, } from "metabase/selectors/app"; import { setErrorPage } from "metabase/redux/app"; import { useOnMount } from "metabase/hooks/use-on-mount"; @@ -52,7 +52,7 @@ interface AppStateProps { isAdminApp: boolean; bannerMessageDescriptor?: string; isAppBarVisible: boolean; - isNavBarVisible: boolean; + isNavBarEnabled: boolean; } interface AppDispatchProps { @@ -73,7 +73,7 @@ const mapStateToProps = ( errorPage: getErrorPage(state), isAdminApp: getIsAdminApp(state, props), isAppBarVisible: getIsAppBarVisible(state, props), - isNavBarVisible: getIsNavBarVisible(state, props), + isNavBarEnabled: getIsNavBarEnabled(state, props), }); const mapDispatchToProps: AppDispatchProps = { @@ -96,7 +96,7 @@ function App({ errorPage, isAdminApp, isAppBarVisible, - isNavBarVisible, + isNavBarEnabled, children, onError, }: AppProps) { @@ -111,9 +111,9 @@ function App({ - {isAppBarVisible && } + {isAppBarVisible && } - {isNavBarVisible && } + {isNavBarEnabled && } {errorPage ? getErrorComponent(errorPage) : children} diff --git a/frontend/src/metabase/account/notifications/containers/NotificationsApp/NotificationsApp.jsx b/frontend/src/metabase/account/notifications/containers/NotificationsApp/NotificationsApp.jsx index 2b54d9c493bb..70a8a21964fa 100644 --- a/frontend/src/metabase/account/notifications/containers/NotificationsApp/NotificationsApp.jsx +++ b/frontend/src/metabase/account/notifications/containers/NotificationsApp/NotificationsApp.jsx @@ -33,7 +33,8 @@ export default _.compose( reload: true, }), Pulses.loadList({ - query: state => ({ user_id: getUserId(state) }), + // Load all pulses the current user can read (i.e. is a creator or recipient of) + query: state => ({ can_read: true }), reload: true, }), connect(mapStateToProps, mapDispatchToProps), diff --git a/frontend/src/metabase/account/password/actions.js b/frontend/src/metabase/account/password/actions.js deleted file mode 100644 index 4041329d863e..000000000000 --- a/frontend/src/metabase/account/password/actions.js +++ /dev/null @@ -1,32 +0,0 @@ -import { t } from "ttag"; -import { UserApi, UtilApi } from "metabase/services"; -import { createThunkAction } from "metabase/lib/redux"; - -export const UPDATE_PASSWORD = "UPDATE_PASSWORD"; -export const VALIDATE_PASSWORD = "VALIDATE_PASSWORD"; - -export const validatePassword = createThunkAction( - VALIDATE_PASSWORD, - password => async () => - UtilApi.password_check({ - password, - }), -); - -export const updatePassword = createThunkAction( - UPDATE_PASSWORD, - (user_id, password, old_password) => async () => { - await UserApi.update_password({ - id: user_id, - password, - old_password, - }); - - return { - success: true, - data: { - message: t`Password updated successfully!`, - }, - }; - }, -); diff --git a/frontend/src/metabase/account/password/actions.ts b/frontend/src/metabase/account/password/actions.ts new file mode 100644 index 000000000000..0ec7da41d473 --- /dev/null +++ b/frontend/src/metabase/account/password/actions.ts @@ -0,0 +1,26 @@ +import { getIn } from "icepick"; +import { UserApi, UtilApi } from "metabase/services"; +import MetabaseSettings from "metabase/lib/settings"; +import { User } from "metabase-types/api"; +import { UserPasswordData } from "./types"; + +export const validatePassword = async (password: string) => { + const error = MetabaseSettings.passwordComplexityDescription(password); + if (error) { + return error; + } + + try { + await UtilApi.password_check({ password }); + } catch (error) { + return getIn(error, ["data", "errors", "password"]); + } +}; + +export const updatePassword = async (user: User, data: UserPasswordData) => { + await UserApi.update_password({ + id: user.id, + password: data.password, + old_password: data.old_password, + }); +}; diff --git a/frontend/src/metabase/account/password/components/UserPasswordForm/UserPasswordForm.jsx b/frontend/src/metabase/account/password/components/UserPasswordForm/UserPasswordForm.jsx deleted file mode 100644 index 08d78013c73a..000000000000 --- a/frontend/src/metabase/account/password/components/UserPasswordForm/UserPasswordForm.jsx +++ /dev/null @@ -1,44 +0,0 @@ -import React, { useCallback } from "react"; -import PropTypes from "prop-types"; -import { t } from "ttag"; -import User from "metabase/entities/users"; - -const propTypes = { - user: PropTypes.object, - validatePassword: PropTypes.func, - updatePassword: PropTypes.func, -}; - -const UserPasswordForm = ({ user, validatePassword, updatePassword }) => { - const handleAsyncValidate = useCallback( - async ({ password }) => { - try { - await validatePassword(password); - } catch (error) { - return error.data.errors; - } - }, - [validatePassword], - ); - - const handleSubmit = useCallback( - async ({ password, old_password }) => { - await updatePassword(user.id, password, old_password); - }, - [user, updatePassword], - ); - - return ( - - ); -}; - -UserPasswordForm.propTypes = propTypes; - -export default UserPasswordForm; diff --git a/frontend/src/metabase/account/password/components/UserPasswordForm/UserPasswordForm.tsx b/frontend/src/metabase/account/password/components/UserPasswordForm/UserPasswordForm.tsx new file mode 100644 index 000000000000..8344c2956373 --- /dev/null +++ b/frontend/src/metabase/account/password/components/UserPasswordForm/UserPasswordForm.tsx @@ -0,0 +1,92 @@ +import React, { useCallback, useMemo } from "react"; +import { t } from "ttag"; +import _ from "underscore"; +import * as Yup from "yup"; +import Form from "metabase/core/components/Form"; +import FormProvider from "metabase/core/components/FormProvider"; +import FormInput from "metabase/core/components/FormInput"; +import FormSubmitButton from "metabase/core/components/FormSubmitButton"; +import FormErrorMessage from "metabase/core/components/FormErrorMessage"; +import * as Errors from "metabase/core/utils/errors"; +import { User } from "metabase-types/api"; +import { UserPasswordData } from "../../types"; + +const USER_PASSWORD_SCHEMA = Yup.object({ + old_password: Yup.string().default("").required(Errors.required), + password: Yup.string() + .default("") + .required(Errors.required) + .test(async (value = "", context) => { + const error = await context.options.context?.onValidatePassword(value); + return error ? context.createError({ message: error }) : true; + }), + password_confirm: Yup.string() + .default("") + .required(Errors.required) + .oneOf([Yup.ref("password")], t`passwords do not match`), +}); + +export interface UserPasswordFormProps { + user: User; + onValidatePassword: (password: string) => Promise; + onSubmit: (user: User, data: UserPasswordData) => void; +} + +const UserPasswordForm = ({ + user, + onValidatePassword, + onSubmit, +}: UserPasswordFormProps): JSX.Element => { + const initialValues = useMemo(() => { + return USER_PASSWORD_SCHEMA.getDefault(); + }, []); + + const validationContext = useMemo( + () => ({ onValidatePassword: _.memoize(onValidatePassword) }), + [onValidatePassword], + ); + + const handleSubmit = useCallback( + (data: UserPasswordData) => { + return onSubmit(user, data); + }, + [user, onSubmit], + ); + + return ( + +
+ + + + + + + + ); +}; + +export default UserPasswordForm; diff --git a/frontend/src/metabase/account/password/components/UserPasswordForm/index.js b/frontend/src/metabase/account/password/components/UserPasswordForm/index.ts similarity index 100% rename from frontend/src/metabase/account/password/components/UserPasswordForm/index.js rename to frontend/src/metabase/account/password/components/UserPasswordForm/index.ts diff --git a/frontend/src/metabase/account/password/containers/UserPasswordApp/UserPasswordApp.jsx b/frontend/src/metabase/account/password/containers/UserPasswordApp/UserPasswordApp.tsx similarity index 53% rename from frontend/src/metabase/account/password/containers/UserPasswordApp/UserPasswordApp.jsx rename to frontend/src/metabase/account/password/containers/UserPasswordApp/UserPasswordApp.tsx index 723ff7997ef2..4e5f026b8159 100644 --- a/frontend/src/metabase/account/password/containers/UserPasswordApp/UserPasswordApp.jsx +++ b/frontend/src/metabase/account/password/containers/UserPasswordApp/UserPasswordApp.tsx @@ -1,15 +1,13 @@ import { connect } from "react-redux"; import { getUser } from "metabase/selectors/user"; +import { State } from "metabase-types/store"; import { updatePassword, validatePassword } from "../../actions"; import UserPasswordForm from "../../components/UserPasswordForm"; -const mapStateToProps = state => ({ +const mapStateToProps = (state: State) => ({ user: getUser(state), + onValidatePassword: validatePassword, + onSubmit: updatePassword, }); -const mapDispatchToProps = { - validatePassword, - updatePassword, -}; - -export default connect(mapStateToProps, mapDispatchToProps)(UserPasswordForm); +export default connect(mapStateToProps)(UserPasswordForm); diff --git a/frontend/src/metabase/account/password/containers/UserPasswordApp/index.js b/frontend/src/metabase/account/password/containers/UserPasswordApp/index.ts similarity index 100% rename from frontend/src/metabase/account/password/containers/UserPasswordApp/index.js rename to frontend/src/metabase/account/password/containers/UserPasswordApp/index.ts diff --git a/frontend/src/metabase/account/password/types.ts b/frontend/src/metabase/account/password/types.ts new file mode 100644 index 000000000000..c48379e2a840 --- /dev/null +++ b/frontend/src/metabase/account/password/types.ts @@ -0,0 +1,5 @@ +export interface UserPasswordData { + old_password: string; + password: string; + password_confirm: string; +} diff --git a/frontend/src/metabase/account/profile/actions.ts b/frontend/src/metabase/account/profile/actions.ts new file mode 100644 index 000000000000..b4756f45851f --- /dev/null +++ b/frontend/src/metabase/account/profile/actions.ts @@ -0,0 +1,17 @@ +import { createThunkAction } from "metabase/lib/redux"; +import Users from "metabase/entities/users"; +import { User } from "metabase-types/api"; +import { Dispatch } from "metabase-types/store"; +import { UserProfileData } from "./types"; + +export const UPDATE_USER = "metabase/account/profile/UPDATE_USER"; +export const updateUser = createThunkAction( + UPDATE_USER, + (user: User, data: UserProfileData) => async (dispatch: Dispatch) => { + await dispatch(Users.actions.update({ ...data, id: user.id })); + + if (user.locale !== data.locale) { + window.location.reload(); + } + }, +); diff --git a/frontend/src/metabase/account/profile/components/UserProfileForm/UserProfileForm.jsx b/frontend/src/metabase/account/profile/components/UserProfileForm/UserProfileForm.jsx deleted file mode 100644 index 4a31103534a7..000000000000 --- a/frontend/src/metabase/account/profile/components/UserProfileForm/UserProfileForm.jsx +++ /dev/null @@ -1,32 +0,0 @@ -import React, { useCallback } from "react"; -import PropTypes from "prop-types"; - -import User from "metabase/entities/users"; - -const propTypes = { - user: PropTypes.object, -}; - -const UserProfileForm = ({ user }) => { - const handleSaved = useCallback( - values => { - if (user.locale !== values.locale) { - window.location.reload(); - } - }, - [user?.locale], - ); - - return ( - - ); -}; - -UserProfileForm.propTypes = propTypes; - -export default UserProfileForm; diff --git a/frontend/src/metabase/account/profile/components/UserProfileForm/UserProfileForm.tsx b/frontend/src/metabase/account/profile/components/UserProfileForm/UserProfileForm.tsx new file mode 100644 index 000000000000..945f5d4e737f --- /dev/null +++ b/frontend/src/metabase/account/profile/components/UserProfileForm/UserProfileForm.tsx @@ -0,0 +1,106 @@ +import React, { useCallback, useMemo } from "react"; +import { t } from "ttag"; +import * as Yup from "yup"; +import _ from "underscore"; +import Form from "metabase/core/components/Form"; +import FormProvider from "metabase/core/components/FormProvider"; +import FormInput from "metabase/core/components/FormInput"; +import FormSelect from "metabase/core/components/FormSelect"; +import FormSubmitButton from "metabase/core/components/FormSubmitButton"; +import FormErrorMessage from "metabase/core/components/FormErrorMessage"; +import * as Errors from "metabase/core/utils/errors"; +import { LocaleData, User } from "metabase-types/api"; +import { UserProfileData } from "../../types"; + +const SSO_PROFILE_SCHEMA = Yup.object({ + locale: Yup.string().nullable().default(null), +}); + +const LOCAL_PROFILE_SCHEMA = SSO_PROFILE_SCHEMA.shape({ + first_name: Yup.string().nullable().default(null).max(100, Errors.maxLength), + last_name: Yup.string().nullable().default(null).max(100, Errors.maxLength), + email: Yup.string().ensure().required(Errors.required).email(Errors.email), +}); + +export interface UserProfileFormProps { + user: User; + locales: LocaleData[] | null; + isSsoUser: boolean; + onSubmit: (user: User, data: UserProfileData) => void; +} + +const UserProfileForm = ({ + user, + locales, + isSsoUser, + onSubmit, +}: UserProfileFormProps): JSX.Element => { + const schema = isSsoUser ? SSO_PROFILE_SCHEMA : LOCAL_PROFILE_SCHEMA; + + const initialValues = useMemo(() => { + return schema.cast(user, { stripUnknown: true }); + }, [user, schema]); + + const localeOptions = useMemo(() => { + return getLocaleOptions(locales); + }, [locales]); + + const handleSubmit = useCallback( + (values: UserProfileData) => onSubmit(user, values), + [user, onSubmit], + ); + + return ( + + {({ dirty }) => ( +
+ {!isSsoUser && ( + <> + + + + + )} + + + + + )} + + ); +}; + +const getLocaleOptions = (locales: LocaleData[] | null) => { + const options = _.chain(locales ?? [["en", "English"]]) + .map(([value, name]) => ({ name, value })) + .sortBy(({ name }) => name) + .value(); + + return [{ name: t`Use site default`, value: null }, ...options]; +}; + +export default UserProfileForm; diff --git a/frontend/src/metabase/account/profile/components/UserProfileForm/UserProfileForm.unit.spec.tsx b/frontend/src/metabase/account/profile/components/UserProfileForm/UserProfileForm.unit.spec.tsx new file mode 100644 index 000000000000..93062cec8fbd --- /dev/null +++ b/frontend/src/metabase/account/profile/components/UserProfileForm/UserProfileForm.unit.spec.tsx @@ -0,0 +1,30 @@ +import React from "react"; +import { render, screen } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { createMockUser } from "metabase-types/api/mocks"; +import UserProfileForm, { UserProfileFormProps } from "./UserProfileForm"; + +describe("UserProfileForm", () => { + it("should show a success message after form submit", async () => { + const props = getProps({ + onSubmit: jest.fn().mockResolvedValue({}), + }); + + render(); + userEvent.clear(screen.getByLabelText("First name")); + userEvent.type(screen.getByLabelText("First name"), "New name"); + userEvent.click(screen.getByText("Update")); + + expect(await screen.findByText("Success")).toBeInTheDocument(); + }); +}); + +const getProps = ( + opts?: Partial, +): UserProfileFormProps => ({ + user: createMockUser(), + locales: null, + isSsoUser: false, + onSubmit: jest.fn(), + ...opts, +}); diff --git a/frontend/src/metabase/account/profile/components/UserProfileForm/index.js b/frontend/src/metabase/account/profile/components/UserProfileForm/index.ts similarity index 100% rename from frontend/src/metabase/account/profile/components/UserProfileForm/index.js rename to frontend/src/metabase/account/profile/components/UserProfileForm/index.ts diff --git a/frontend/src/metabase/account/profile/containers/UserProfileApp/UserProfileApp.jsx b/frontend/src/metabase/account/profile/containers/UserProfileApp/UserProfileApp.jsx deleted file mode 100644 index 5b26c02d115d..000000000000 --- a/frontend/src/metabase/account/profile/containers/UserProfileApp/UserProfileApp.jsx +++ /dev/null @@ -1,9 +0,0 @@ -import { connect } from "react-redux"; -import { getUser } from "metabase/selectors/user"; -import UserProfileForm from "../../components/UserProfileForm"; - -const mapStateToProps = state => ({ - user: getUser(state), -}); - -export default connect(mapStateToProps)(UserProfileForm); diff --git a/frontend/src/metabase/account/profile/containers/UserProfileApp/UserProfileApp.tsx b/frontend/src/metabase/account/profile/containers/UserProfileApp/UserProfileApp.tsx new file mode 100644 index 000000000000..dbde8a6ed261 --- /dev/null +++ b/frontend/src/metabase/account/profile/containers/UserProfileApp/UserProfileApp.tsx @@ -0,0 +1,18 @@ +import { connect } from "react-redux"; +import { getUser } from "metabase/selectors/user"; +import { State } from "metabase-types/store"; +import UserProfileForm from "../../components/UserProfileForm"; +import { updateUser } from "../../actions"; +import { getIsSsoUser, getLocales } from "../../selectors"; + +const mapStateToProps = (state: State) => ({ + user: getUser(state), + locales: getLocales(state), + isSsoUser: getIsSsoUser(state), +}); + +const mapDispatchToProps = { + onSubmit: updateUser, +}; + +export default connect(mapStateToProps, mapDispatchToProps)(UserProfileForm); diff --git a/frontend/src/metabase/account/profile/containers/UserProfileApp/index.js b/frontend/src/metabase/account/profile/containers/UserProfileApp/index.ts similarity index 100% rename from frontend/src/metabase/account/profile/containers/UserProfileApp/index.js rename to frontend/src/metabase/account/profile/containers/UserProfileApp/index.ts diff --git a/frontend/src/metabase/account/profile/selectors.ts b/frontend/src/metabase/account/profile/selectors.ts new file mode 100644 index 000000000000..46c12d573cc5 --- /dev/null +++ b/frontend/src/metabase/account/profile/selectors.ts @@ -0,0 +1,12 @@ +import { createSelector } from "reselect"; +import { getUser } from "metabase/selectors/user"; +import { PLUGIN_IS_PASSWORD_USER } from "metabase/plugins"; +import { getSettings } from "metabase/selectors/settings"; + +export const getIsSsoUser = createSelector([getUser], user => { + return !PLUGIN_IS_PASSWORD_USER.every(predicate => predicate(user)); +}); + +export const getLocales = createSelector([getSettings], settings => { + return settings["available-locales"]; +}); diff --git a/frontend/src/metabase/account/profile/types.ts b/frontend/src/metabase/account/profile/types.ts new file mode 100644 index 000000000000..3fbc02ce4eac --- /dev/null +++ b/frontend/src/metabase/account/profile/types.ts @@ -0,0 +1,6 @@ +export interface UserProfileData { + first_name?: string | null; + last_name?: string | null; + email?: string; + locale: string | null; +} diff --git a/frontend/src/metabase/admin/app/components/DeprecationNotice/DeprecationNotice.stories.tsx b/frontend/src/metabase/admin/app/components/DeprecationNotice/DeprecationNotice.stories.tsx index e7274f919b1d..81339eef0e0d 100644 --- a/frontend/src/metabase/admin/app/components/DeprecationNotice/DeprecationNotice.stories.tsx +++ b/frontend/src/metabase/admin/app/components/DeprecationNotice/DeprecationNotice.stories.tsx @@ -1,5 +1,5 @@ import React from "react"; -import { ComponentStory } from "@storybook/react"; +import type { ComponentStory } from "@storybook/react"; import DeprecationNotice from "./DeprecationNotice"; export default { diff --git a/frontend/src/metabase/admin/datamodel/containers/DataModelApp/ModelEducationalModal.jsx b/frontend/src/metabase/admin/datamodel/containers/DataModelApp/ModelEducationalModal.jsx index 3d1993a8a8c6..089cdf9c7b36 100644 --- a/frontend/src/metabase/admin/datamodel/containers/DataModelApp/ModelEducationalModal.jsx +++ b/frontend/src/metabase/admin/datamodel/containers/DataModelApp/ModelEducationalModal.jsx @@ -1,10 +1,9 @@ import React from "react"; import PropTypes from "prop-types"; import { t } from "ttag"; - +import MetabaseSettings from "metabase/lib/settings"; import Modal from "metabase/components/Modal"; import ModalContent from "metabase/components/ModalContent"; - import { Content, Description, @@ -17,7 +16,7 @@ ModelEducationalModal.propTypes = { onClose: PropTypes.func.isRequired, }; -const EDUCATION_URL = "https://metabase.com/learn/getting-started/models"; +const EDUCATION_URL = MetabaseSettings.learnUrl("getting-started/models"); export function ModelEducationalModal({ isOpen, onClose }) { return ( diff --git a/frontend/src/metabase/admin/settings/auth/components/AuthCard/AuthCard.styled.tsx b/frontend/src/metabase/admin/settings/auth/components/AuthCard/AuthCard.styled.tsx new file mode 100644 index 000000000000..4021c8aa5753 --- /dev/null +++ b/frontend/src/metabase/admin/settings/auth/components/AuthCard/AuthCard.styled.tsx @@ -0,0 +1,50 @@ +import styled from "@emotion/styled"; +import { color } from "metabase/lib/colors"; +import EntityMenu from "metabase/components/EntityMenu"; + +export const CardRoot = styled.div` + width: 31.25rem; + padding: 2rem; + border: 1px solid ${color("border")}; + border-radius: 0.5rem; + box-shadow: 0 2px 2px ${color("shadow")}; + background-color: ${color("white")}; +`; + +export const CardHeader = styled.div` + display: flex; + align-items: center; + gap: 0.5rem; + margin-bottom: 1rem; +`; + +export const CardTitle = styled.div` + color: ${color("text-dark")}; + font-size: 1.5rem; + line-height: 2rem; + font-weight: bold; +`; + +export const CardDescription = styled.div` + color: ${color("text-dark")}; + font-size: 0.875rem; + line-height: 1.5rem; + margin-bottom: 1.5rem; +`; + +interface CardBadgeProps { + isEnabled: boolean; +} + +export const CardBadge = styled.div` + color: ${props => color(props.isEnabled ? "brand" : "danger")}; + background-color: ${props => + color(props.isEnabled ? "brand-light" : "bg-light")}; + padding: 0.25rem 0.375rem; + border-radius: 0.25rem; + font-weight: bold; +`; + +export const CardMenu = styled(EntityMenu)` + margin-left: auto; +`; diff --git a/frontend/src/metabase/admin/settings/auth/components/AuthCard/AuthCard.tsx b/frontend/src/metabase/admin/settings/auth/components/AuthCard/AuthCard.tsx new file mode 100644 index 000000000000..250db513b7f6 --- /dev/null +++ b/frontend/src/metabase/admin/settings/auth/components/AuthCard/AuthCard.tsx @@ -0,0 +1,177 @@ +import React, { ReactNode, useCallback, useMemo, useState } from "react"; +import { t } from "ttag"; +import { Link } from "react-router"; +import Button from "metabase/core/components/Button"; +import Modal from "metabase/components/Modal"; +import ModalContent from "metabase/components/ModalContent"; +import { + CardBadge, + CardDescription, + CardHeader, + CardMenu, + CardRoot, + CardTitle, +} from "./AuthCard.styled"; + +export interface AuthSetting { + value: boolean | null; +} + +export interface AuthCardProps { + setting: AuthSetting; + type: string; + name: string; + title?: string; + description: string; + isConfigured: boolean; + onChange: (value: boolean) => void; + onDeactivate: () => void; +} + +const AuthCard = ({ + setting, + type, + name, + title = name, + description, + isConfigured, + onChange, + onDeactivate, +}: AuthCardProps) => { + const isEnabled = setting.value ?? false; + const [isOpened, setIsOpened] = useState(false); + + const handleOpen = useCallback(() => { + setIsOpened(true); + }, []); + + const handleClose = useCallback(() => { + setIsOpened(false); + }, []); + + const handleDeactivate = useCallback(async () => { + await onDeactivate(); + handleClose(); + }, [onDeactivate, handleClose]); + + return ( + + {isConfigured && ( + + )} + {isOpened && ( + + )} + + ); +}; + +interface AuthCardBodyProps { + type: string; + title: string; + description: string; + isEnabled: boolean; + isConfigured: boolean; + children?: ReactNode; +} + +const AuthCardBody = ({ + type, + title, + description, + isEnabled, + isConfigured, + children, +}: AuthCardBodyProps) => { + return ( + + + {title} + {isConfigured && ( + + {isEnabled ? t`Active` : t`Paused`} + + )} + {children} + + {description} + + + ); +}; + +interface AuthCardMenuProps { + isEnabled: boolean; + onChange: (isEnabled: boolean) => void; + onDeactivate: () => void; +} + +const AuthCardMenu = ({ + isEnabled, + onChange, + onDeactivate, +}: AuthCardMenuProps): JSX.Element => { + const menuItems = useMemo( + () => [ + { + title: isEnabled ? t`Pause` : t`Resume`, + icon: isEnabled ? "pause" : "play", + action: () => onChange(!isEnabled), + }, + { + title: `Deactivate`, + icon: "close", + action: onDeactivate, + }, + ], + [isEnabled, onChange, onDeactivate], + ); + + return ; +}; + +interface AuthCardModalProps { + name: string; + onDeactivate: () => void; + onClose: () => void; +} + +const AuthCardModal = ({ + name, + onDeactivate, + onClose, +}: AuthCardModalProps): JSX.Element => { + return ( + + {t`Cancel`}, + , + ]} + > + {t`This will clear all your settings.`} + + + ); +}; + +export default AuthCard; diff --git a/frontend/src/metabase/admin/settings/auth/components/AuthCard/AuthCard.unit.spec.tsx b/frontend/src/metabase/admin/settings/auth/components/AuthCard/AuthCard.unit.spec.tsx new file mode 100644 index 000000000000..009700c77bf7 --- /dev/null +++ b/frontend/src/metabase/admin/settings/auth/components/AuthCard/AuthCard.unit.spec.tsx @@ -0,0 +1,73 @@ +import React from "react"; +import { render, screen } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import AuthCard, { AuthSetting, AuthCardProps } from "./AuthCard"; + +describe("AuthCard", () => { + it("should render when not configured", () => { + const props = getProps({ + isConfigured: false, + }); + + render(); + + expect(screen.getByText("Set up")).toBeInTheDocument(); + expect(screen.queryByLabelText("ellipsis icon")).not.toBeInTheDocument(); + }); + + it("should pause active authentication", () => { + const props = getProps({ + setting: getSetting({ value: true }), + isConfigured: true, + }); + + render(); + userEvent.click(screen.getByLabelText("ellipsis icon")); + userEvent.click(screen.getByText("Pause")); + + expect(props.onChange).toHaveBeenCalledWith(false); + }); + + it("should resume paused authentication", () => { + const props = getProps({ + setting: getSetting({ value: false }), + isConfigured: true, + }); + + render(); + userEvent.click(screen.getByLabelText("ellipsis icon")); + userEvent.click(screen.getByText("Resume")); + + expect(props.onChange).toHaveBeenCalledWith(true); + }); + + it("should deactivate authentication", () => { + const props = getProps({ + setting: getSetting({ value: false }), + isConfigured: true, + }); + + render(); + userEvent.click(screen.getByLabelText("ellipsis icon")); + userEvent.click(screen.getByText("Deactivate")); + userEvent.click(screen.getByRole("button", { name: "Deactivate" })); + + expect(props.onDeactivate).toHaveBeenCalled(); + }); +}); + +const getSetting = (opts?: Partial): AuthSetting => ({ + value: false, + ...opts, +}); + +const getProps = (opts?: Partial): AuthCardProps => ({ + setting: getSetting(), + type: "type", + name: "SSO", + description: "SSO authentication", + isConfigured: false, + onChange: jest.fn(), + onDeactivate: jest.fn(), + ...opts, +}); diff --git a/frontend/src/metabase/admin/settings/auth/components/AuthCard/index.ts b/frontend/src/metabase/admin/settings/auth/components/AuthCard/index.ts new file mode 100644 index 000000000000..be8963d01894 --- /dev/null +++ b/frontend/src/metabase/admin/settings/auth/components/AuthCard/index.ts @@ -0,0 +1,2 @@ +export { default } from "./AuthCard"; +export type { AuthCardProps } from "./AuthCard"; diff --git a/frontend/src/metabase/admin/settings/auth/constants.ts b/frontend/src/metabase/admin/settings/auth/constants.ts new file mode 100644 index 000000000000..fdb15c256847 --- /dev/null +++ b/frontend/src/metabase/admin/settings/auth/constants.ts @@ -0,0 +1,26 @@ +import * as Yup from "yup"; + +export const GOOGLE_SCHEMA = Yup.object({ + "google-auth-enabled": Yup.boolean().default(false), + "google-auth-client-id": Yup.string().nullable().default(null), + "google-auth-auto-create-accounts-domain": Yup.string() + .nullable() + .default(null), +}); + +export const LDAP_SCHEMA = Yup.object({ + "ldap-enabled": Yup.boolean().default(false), + "ldap-host": Yup.string().nullable().default(null), + "ldap-port": Yup.number().nullable().default(null), + "ldap-security": Yup.string().default("none"), + "ldap-bind-dn": Yup.string().nullable().default(null), + "ldap-password": Yup.string().nullable().default(null), + "ldap-user-base": Yup.string().nullable().default(null), + "ldap-user-filter": Yup.string().nullable().default(null), + "ldap-attribute-email": Yup.string().nullable().default(null), + "ldap-attribute-firstname": Yup.string().nullable().default(null), + "ldap-attribute-lastname": Yup.string().nullable().default(null), + "ldap-group-sync": Yup.boolean().default(false), + "ldap-group-base": Yup.string().nullable().default(null), + "ldap-group-mappings": Yup.object().default(null), +}); diff --git a/frontend/src/metabase/admin/settings/auth/containers/GoogleAuthCard/GoogleAuthCard.tsx b/frontend/src/metabase/admin/settings/auth/containers/GoogleAuthCard/GoogleAuthCard.tsx new file mode 100644 index 000000000000..52f1b94d8590 --- /dev/null +++ b/frontend/src/metabase/admin/settings/auth/containers/GoogleAuthCard/GoogleAuthCard.tsx @@ -0,0 +1,24 @@ +import { t } from "ttag"; +import { connect } from "react-redux"; +import { getSetting } from "metabase/selectors/settings"; +import { updateSettings } from "metabase/admin/settings/settings"; +import { Dispatch, State } from "metabase-types/store"; +import AuthCard, { AuthCardProps } from "../../components/AuthCard"; +import { GOOGLE_SCHEMA } from "../../constants"; + +type StateProps = Omit; +type DispatchProps = Pick; + +const mapStateToProps = (state: State): StateProps => ({ + type: "google", + name: t`Google Sign-in`, + title: t`Sign in with Google`, + description: t`Allows users with existing Metabase accounts to login with a Google account that matches their email address in addition to their Metabase username and password.`, + isConfigured: getSetting(state, "google-auth-configured"), +}); + +const mapDispatchToProps = (dispatch: Dispatch): DispatchProps => ({ + onDeactivate: () => dispatch(updateSettings(GOOGLE_SCHEMA.getDefault())), +}); + +export default connect(mapStateToProps, mapDispatchToProps)(AuthCard); diff --git a/frontend/src/metabase/admin/settings/auth/containers/GoogleAuthCard/index.ts b/frontend/src/metabase/admin/settings/auth/containers/GoogleAuthCard/index.ts new file mode 100644 index 000000000000..7cc86d923fe3 --- /dev/null +++ b/frontend/src/metabase/admin/settings/auth/containers/GoogleAuthCard/index.ts @@ -0,0 +1 @@ +export { default } from "./GoogleAuthCard"; diff --git a/frontend/src/metabase/admin/settings/auth/containers/LdapAuthCard/LdapAuthCard.tsx b/frontend/src/metabase/admin/settings/auth/containers/LdapAuthCard/LdapAuthCard.tsx new file mode 100644 index 000000000000..1d396a62c282 --- /dev/null +++ b/frontend/src/metabase/admin/settings/auth/containers/LdapAuthCard/LdapAuthCard.tsx @@ -0,0 +1,20 @@ +import { t } from "ttag"; +import { connect } from "react-redux"; +import { getSetting } from "metabase/selectors/settings"; +import { updateSettings } from "metabase/admin/settings/settings"; +import { Dispatch, State } from "metabase-types/store"; +import AuthCard from "../../components/AuthCard"; +import { LDAP_SCHEMA } from "../../constants"; + +const mapStateToProps = (state: State) => ({ + type: "ldap", + name: t`LDAP`, + description: t`Allows users within your LDAP directory to log in to Metabase with their LDAP credentials, and allows automatic mapping of LDAP groups to Metabase groups.`, + isConfigured: getSetting(state, "ldap-configured?"), +}); + +const mapDispatchToProps = (dispatch: Dispatch) => ({ + onDeactivate: () => dispatch(updateSettings(LDAP_SCHEMA.getDefault())), +}); + +export default connect(mapStateToProps, mapDispatchToProps)(AuthCard); diff --git a/frontend/src/metabase/admin/settings/auth/containers/LdapAuthCard/index.ts b/frontend/src/metabase/admin/settings/auth/containers/LdapAuthCard/index.ts new file mode 100644 index 000000000000..27c44364f9f0 --- /dev/null +++ b/frontend/src/metabase/admin/settings/auth/containers/LdapAuthCard/index.ts @@ -0,0 +1 @@ +export { default } from "./LdapAuthCard"; diff --git a/frontend/src/metabase/admin/settings/components/SettingsBatchForm.jsx b/frontend/src/metabase/admin/settings/components/SettingsBatchForm.jsx index 97db374b4c46..b484e154c522 100644 --- a/frontend/src/metabase/admin/settings/components/SettingsBatchForm.jsx +++ b/frontend/src/metabase/admin/settings/components/SettingsBatchForm.jsx @@ -188,7 +188,8 @@ class SettingsBatchForm extends Component { return formErrors; } - handleSubmit = updateSettings => { + handleSubmit = () => { + const { updateSettings } = this.props; const { formData, valid } = this.state; if (valid) { @@ -217,7 +218,7 @@ class SettingsBatchForm extends Component { handleSubmitClick = event => { event.preventDefault(); - this.handleSubmit(this.props.updateSettings); + this.handleSubmit(); }; render() { diff --git a/frontend/src/metabase/admin/settings/components/SettingsGoogleForm/SettingsGoogleForm.tsx b/frontend/src/metabase/admin/settings/components/SettingsGoogleForm/SettingsGoogleForm.tsx index 6f4dbbb46020..7dfcd355b337 100644 --- a/frontend/src/metabase/admin/settings/components/SettingsGoogleForm/SettingsGoogleForm.tsx +++ b/frontend/src/metabase/admin/settings/components/SettingsGoogleForm/SettingsGoogleForm.tsx @@ -1,8 +1,7 @@ -import React, { useRef } from "react"; +import React, { useCallback } from "react"; import { connect } from "react-redux"; import { jt, t } from "ttag"; import MetabaseSettings from "metabase/lib/settings"; -import ActionButton from "metabase/components/ActionButton"; import ExternalLink from "metabase/core/components/ExternalLink"; import Breadcrumbs from "metabase/components/Breadcrumbs"; import { @@ -35,44 +34,17 @@ const SettingsGoogleForm = ({ onSubmit, }: SettingsGoogleFormProps) => { const isEnabled = Boolean(settingValues["google-auth-enabled"]); - const isEnabledRef = useRef(isEnabled); - const handleSubmit = (values: Record) => { - return onSubmit({ ...values, "google-auth-enabled": isEnabledRef.current }); - }; - - const handleSaveAndEnableClick = (handleSubmit: () => void) => { - isEnabledRef.current = true; - return handleSubmit(); - }; - - const handleSaveAndNotEnableClick = (handleSubmit: () => void) => { - isEnabledRef.current = false; - return handleSubmit(); - }; + const handleSubmit = useCallback( + (values: Record) => { + return onSubmit({ ...values, "google-auth-enabled": true }); + }, + [onSubmit], + ); return ( ( - <> - handleSaveAndEnableClick(handleSubmit)} - primary={canSubmit} - disabled={!canSubmit} - normalText={isEnabled ? t`Save changes` : t`Save and enable`} - successText={t`Changes saved!`} - /> - {!isEnabled && ( - handleSaveAndNotEnableClick(handleSubmit)} - disabled={!canSubmit} - normalText={t`Save but don't enable`} - successText={t`Changes saved!`} - /> - )} - - )} disablePristineSubmit overwriteOnInitialValuesChange onSubmit={handleSubmit} @@ -108,7 +80,9 @@ const SettingsGoogleForm = ({ - {t`Save changes`} + + {isEnabled ? t`Save changes` : t`Save and enable`} + ); diff --git a/frontend/src/metabase/admin/settings/components/SettingsLdapForm.jsx b/frontend/src/metabase/admin/settings/components/SettingsLdapForm.jsx index 96482187ed84..031f96520b13 100644 --- a/frontend/src/metabase/admin/settings/components/SettingsLdapForm.jsx +++ b/frontend/src/metabase/admin/settings/components/SettingsLdapForm.jsx @@ -1,4 +1,4 @@ -import React from "react"; +import React, { useCallback } from "react"; import PropTypes from "prop-types"; import { t } from "ttag"; import { connect } from "react-redux"; @@ -8,17 +8,20 @@ import { FormButton } from "./SettingsLdapForm.styled"; const propTypes = { settingValues: PropTypes.object.isRequired, - updateLdapSettings: PropTypes.func.isRequired, + onSubmit: PropTypes.func.isRequired, }; -const SettingsLdapForm = ({ settingValues, updateLdapSettings, ...props }) => { +const SettingsLdapForm = ({ settingValues, onSubmit, ...props }) => { const isEnabled = settingValues["ldap-enabled"]; const layout = getLayout(settingValues); const breadcrumbs = getBreadcrumbs(); - const handleAutoEnableSubmit = formData => { - return updateLdapSettings({ ...formData, "ldap-enabled": true }); - }; + const handleSubmit = useCallback( + values => { + return onSubmit({ ...values, "ldap-enabled": true }); + }, + [onSubmit], + ); return ( { layout={layout} breadcrumbs={breadcrumbs} settingValues={settingValues} - updateSettings={updateLdapSettings} - renderSubmitButton={ - !isEnabled && - (({ disabled, pristine, onSubmit }) => ( - onSubmit(handleAutoEnableSubmit)} - normalText={t`Save and enable`} - successText={t`Changes saved!`} - /> - )) - } - renderExtraButtons={ - !isEnabled && - (({ disabled, pristine, onSubmit }) => ( - onSubmit(updateLdapSettings)} - normalText={t`Save but don't enable`} - successText={t`Changes saved!`} - /> - )) - } + updateSettings={handleSubmit} + renderSubmitButton={({ disabled, pristine, onSubmit }) => ( + + )} /> ); }; @@ -101,6 +90,8 @@ const getBreadcrumbs = () => { return [[t`Authentication`, "/admin/settings/authentication"], [t`LDAP`]]; }; -const mapDispatchToProps = { updateLdapSettings }; +const mapDispatchToProps = { + onSubmit: updateLdapSettings, +}; export default connect(null, mapDispatchToProps)(SettingsLdapForm); diff --git a/frontend/src/metabase/admin/settings/components/SettingsSetting.jsx b/frontend/src/metabase/admin/settings/components/SettingsSetting.jsx index 7da85dc14cca..40deaaa1275a 100644 --- a/frontend/src/metabase/admin/settings/components/SettingsSetting.jsx +++ b/frontend/src/metabase/admin/settings/components/SettingsSetting.jsx @@ -63,7 +63,7 @@ export default class SettingsSetting extends Component { return ( // TODO - this formatting needs to be moved outside this component - + {!setting.noHeader && ( )} diff --git a/frontend/src/metabase/admin/settings/components/widgets/AuthenticationOption.jsx b/frontend/src/metabase/admin/settings/components/widgets/AuthenticationOption.jsx deleted file mode 100644 index d367cfa60c38..000000000000 --- a/frontend/src/metabase/admin/settings/components/widgets/AuthenticationOption.jsx +++ /dev/null @@ -1,30 +0,0 @@ -/* eslint-disable react/prop-types */ -import React from "react"; -import { Link } from "react-router"; -import { t } from "ttag"; - -const AuthenticationOption = ({ setting, settingValues }) => ( -
-
-

{setting.authName}

- {setting.authEnabled && setting.authEnabled(settingValues) && ( -
-
- {t`Active`} -
- )} -
-

{setting.authDescription}

- - {t`Configure`} - -
-); - -export default AuthenticationOption; diff --git a/frontend/src/metabase/admin/settings/components/widgets/AuthenticationWidget/AuthenticationWidget.styled.tsx b/frontend/src/metabase/admin/settings/components/widgets/AuthenticationWidget/AuthenticationWidget.styled.tsx deleted file mode 100644 index cbddeed5fa3b..000000000000 --- a/frontend/src/metabase/admin/settings/components/widgets/AuthenticationWidget/AuthenticationWidget.styled.tsx +++ /dev/null @@ -1,31 +0,0 @@ -import styled from "@emotion/styled"; -import { color } from "metabase/lib/colors"; - -export const WidgetRoot = styled.div` - width: 31.25rem; - padding: 2rem; - border: 1px solid ${color("border")}; - border-radius: 0.5rem; - box-shadow: 0 2px 2px ${color("shadow")}; - background-color: ${color("white")}; -`; - -export const WidgetHeader = styled.div` - display: flex; - justify-content: space-between; - align-items: center; - margin-bottom: 1rem; -`; - -export const WidgetTitle = styled.div` - color: ${color("text-dark")}; - font-size: 1.5rem; - font-weight: bold; -`; - -export const WidgetDescription = styled.div` - color: ${color("text-dark")}; - font-size: 0.875rem; - line-height: 1.5rem; - margin-bottom: 1.5rem; -`; diff --git a/frontend/src/metabase/admin/settings/components/widgets/AuthenticationWidget/AuthenticationWidget.tsx b/frontend/src/metabase/admin/settings/components/widgets/AuthenticationWidget/AuthenticationWidget.tsx deleted file mode 100644 index 14348404f08d..000000000000 --- a/frontend/src/metabase/admin/settings/components/widgets/AuthenticationWidget/AuthenticationWidget.tsx +++ /dev/null @@ -1,53 +0,0 @@ -import React from "react"; -import { t } from "ttag"; -import { Link } from "react-router"; -import Button from "metabase/core/components/Button"; -import Toggle from "metabase/core/components/Toggle"; -import { - WidgetDescription, - WidgetHeader, - WidgetRoot, - WidgetTitle, -} from "./AuthenticationWidget.styled"; - -export interface AuthenticationSetting { - value: boolean | null; - default: boolean; -} - -export interface AuthenticationWidgetProps { - setting: AuthenticationSetting; - authType: string; - authName: string; - authDescription: string; - authConfigured: boolean; - onChange: (value: boolean) => void; -} - -const AuthenticationWidget = ({ - setting, - authType, - authName, - authDescription, - authConfigured, - onChange, -}: AuthenticationWidgetProps) => { - const value = setting.value ?? setting.default; - - return ( - - - {authName} - {authConfigured && ( - - )} - - {authDescription} - - - ); -}; - -export default AuthenticationWidget; diff --git a/frontend/src/metabase/admin/settings/components/widgets/AuthenticationWidget/index.ts b/frontend/src/metabase/admin/settings/components/widgets/AuthenticationWidget/index.ts deleted file mode 100644 index ff7f7bb0bc26..000000000000 --- a/frontend/src/metabase/admin/settings/components/widgets/AuthenticationWidget/index.ts +++ /dev/null @@ -1 +0,0 @@ -export { default } from "./AuthenticationWidget"; diff --git a/frontend/src/metabase/admin/settings/components/widgets/CustomGeoJSONWidget.jsx b/frontend/src/metabase/admin/settings/components/widgets/CustomGeoJSONWidget.jsx index d8276a0811b7..e1c3003676c1 100644 --- a/frontend/src/metabase/admin/settings/components/widgets/CustomGeoJSONWidget.jsx +++ b/frontend/src/metabase/admin/settings/components/widgets/CustomGeoJSONWidget.jsx @@ -317,7 +317,7 @@ const EditMap = ({ onCancel, onSave, }) => ( -
+

{!originalMap ? t`Add a new map` : t`Edit map`}

diff --git a/frontend/src/metabase/admin/settings/components/widgets/EmbeddingLegalese/EmbeddingLegalese.styled.ts b/frontend/src/metabase/admin/settings/components/widgets/EmbeddingLegalese/EmbeddingLegalese.styled.ts index 3bb7d9e5f40f..00f847d15c0a 100644 --- a/frontend/src/metabase/admin/settings/components/widgets/EmbeddingLegalese/EmbeddingLegalese.styled.ts +++ b/frontend/src/metabase/admin/settings/components/widgets/EmbeddingLegalese/EmbeddingLegalese.styled.ts @@ -1,4 +1,3 @@ -import css from "@emotion/css"; import styled from "@emotion/styled"; import CollapseSection from "metabase/components/CollapseSection"; import { HeaderContainer } from "metabase/components/CollapseSection/CollapseSection.styled"; diff --git a/frontend/src/metabase/admin/settings/selectors.js b/frontend/src/metabase/admin/settings/selectors.js index 473a774a8762..e6548857476b 100644 --- a/frontend/src/metabase/admin/settings/selectors.js +++ b/frontend/src/metabase/admin/settings/selectors.js @@ -359,7 +359,9 @@ const SECTIONS = updateSectionsWithPlugins({ description: jt`Allow questions, dashboards, and more to be embedded. ${( {t`Learn more.`} diff --git a/frontend/src/metabase/admin/settings/settings.js b/frontend/src/metabase/admin/settings/settings.js index e97ba898e0ce..e4a75ca2177d 100644 --- a/frontend/src/metabase/admin/settings/settings.js +++ b/frontend/src/metabase/admin/settings/settings.js @@ -124,15 +124,10 @@ export const UPDATE_SLACK_SETTINGS = export const updateSlackSettings = createThunkAction( UPDATE_SLACK_SETTINGS, function (settings) { - return async function (dispatch, getState) { - try { - const result = await SlackApi.updateSettings(settings); - await dispatch(reloadSettings()); - return result; - } catch (error) { - console.log("error updating slack settings", settings, error); - throw error; - } + return async function (dispatch) { + const result = await SlackApi.updateSettings(settings); + await dispatch(reloadSettings()); + return result; }; }, {}, @@ -143,15 +138,10 @@ export const UPDATE_LDAP_SETTINGS = export const updateLdapSettings = createThunkAction( UPDATE_LDAP_SETTINGS, function (settings) { - return async function (dispatch, getState) { - try { - const result = await LdapApi.updateSettings(settings); - await dispatch(reloadSettings()); - return result; - } catch (error) { - console.log("error updating LDAP settings", settings, error); - throw error; - } + return async function (dispatch) { + const result = await LdapApi.updateSettings(settings); + await dispatch(reloadSettings()); + return result; }; }, ); @@ -161,15 +151,10 @@ export const UPDATE_GOOGLE_SETTINGS = export const updateGoogleSettings = createThunkAction( UPDATE_GOOGLE_SETTINGS, function (settings) { - return async function (dispatch, getState) { - try { - const result = await GoogleApi.updateSettings(settings); - await dispatch(reloadSettings()); - return result; - } catch (error) { - console.log("error updating Google settings", settings, error); - throw error; - } + return async function (dispatch) { + const result = await GoogleApi.updateSettings(settings); + await dispatch(reloadSettings()); + return result; }; }, ); diff --git a/frontend/src/metabase/admin/settings/slack/components/SlackForm/SlackForm.styled.tsx b/frontend/src/metabase/admin/settings/slack/components/SlackForm/SlackForm.styled.tsx new file mode 100644 index 000000000000..de4aa58d8b17 --- /dev/null +++ b/frontend/src/metabase/admin/settings/slack/components/SlackForm/SlackForm.styled.tsx @@ -0,0 +1,7 @@ +import styled from "@emotion/styled"; +import { color } from "metabase/lib/colors"; + +export const SlackFormMessage = styled.div` + color: ${color("text-medium")}; + margin: 2rem 0 1rem; +`; diff --git a/frontend/src/metabase/admin/settings/slack/components/SlackForm/SlackForm.tsx b/frontend/src/metabase/admin/settings/slack/components/SlackForm/SlackForm.tsx new file mode 100644 index 000000000000..153420fd6d1b --- /dev/null +++ b/frontend/src/metabase/admin/settings/slack/components/SlackForm/SlackForm.tsx @@ -0,0 +1,76 @@ +import React, { useCallback, useMemo } from "react"; +import { t } from "ttag"; +import * as Yup from "yup"; +import Form from "metabase/core/components/Form"; +import FormProvider from "metabase/core/components/FormProvider"; +import FormInput from "metabase/core/components/FormInput"; +import FormSubmitButton from "metabase/core/components/FormSubmitButton"; +import FormErrorMessage from "metabase/core/components/FormErrorMessage"; +import * as Errors from "metabase/core/utils/errors"; +import { SlackSettings } from "metabase-types/api"; +import { SlackFormMessage } from "./SlackForm.styled"; + +const SLACK_SCHEMA = Yup.object({ + "slack-app-token": Yup.string().ensure().required(Errors.required), + "slack-files-channel": Yup.string() + .ensure() + .required(Errors.required) + .lowercase(), +}); + +export interface SlackFormProps { + initialValues: SlackSettings; + isReadOnly?: boolean; + onSubmit?: (values: SlackSettings) => void; +} + +const SlackForm = ({ + initialValues, + isReadOnly, + onSubmit = () => undefined, +}: SlackFormProps): JSX.Element => { + const handleSubmit = useCallback( + (values: SlackSettings) => onSubmit(SLACK_SCHEMA.cast(values)), + [onSubmit], + ); + + return ( + +
+ + {!isReadOnly && ( + + {SLACK_CHANNEL_PROMPT} {SLACK_CHANNEL_DESCRIPTION} + + )} + + {!isReadOnly && ( + <> + + + + )} + + + ); +}; + +const SLACK_CHANNEL_PROMPT = t`Finally, open Slack, create a public channel and enter its name below.`; +const SLACK_CHANNEL_DESCRIPTION = t`This channel shouldn't really be used by anyone — we'll upload charts and tables here before sending out dashboard subscriptions (it's a Slack requirement).`; + +export default SlackForm; diff --git a/frontend/src/metabase/admin/settings/slack/components/SlackForm/index.ts b/frontend/src/metabase/admin/settings/slack/components/SlackForm/index.ts new file mode 100644 index 000000000000..7f69ad530433 --- /dev/null +++ b/frontend/src/metabase/admin/settings/slack/components/SlackForm/index.ts @@ -0,0 +1 @@ +export { default } from "./SlackForm"; diff --git a/frontend/src/metabase/admin/settings/slack/components/SlackSetup/SlackSetup.stories.tsx b/frontend/src/metabase/admin/settings/slack/components/SlackSetup/SlackSetup.stories.tsx index 663dc5e7c77e..9c8fd1a895ac 100644 --- a/frontend/src/metabase/admin/settings/slack/components/SlackSetup/SlackSetup.stories.tsx +++ b/frontend/src/metabase/admin/settings/slack/components/SlackSetup/SlackSetup.stories.tsx @@ -1,5 +1,5 @@ import React from "react"; -import { ComponentStory } from "@storybook/react"; +import type { ComponentStory } from "@storybook/react"; import SlackSetup from "./SlackSetup"; export default { diff --git a/frontend/src/metabase/admin/settings/slack/components/SlackSetupForm/SlackSetupForm.tsx b/frontend/src/metabase/admin/settings/slack/components/SlackSetupForm/SlackSetupForm.tsx index 40849a19e4a6..b36a86be285f 100644 --- a/frontend/src/metabase/admin/settings/slack/components/SlackSetupForm/SlackSetupForm.tsx +++ b/frontend/src/metabase/admin/settings/slack/components/SlackSetupForm/SlackSetupForm.tsx @@ -1,33 +1,18 @@ -import React, { useMemo } from "react"; -import { t } from "ttag"; -import Form from "metabase/containers/FormikForm"; +import React from "react"; import { SlackSettings } from "metabase-types/api"; -import { getSlackForm } from "../../forms"; -import { FormProps } from "./types"; -import { FormMessage } from "./SlackSetupForm.styled"; +import SlackForm from "../SlackForm"; + +const DEFAULT_SETTINGS: SlackSettings = { + "slack-app-token": "", + "slack-files-channel": "", +}; export interface SlackSetupFormProps { onSubmit: (settings: SlackSettings) => void; } const SlackSetupForm = ({ onSubmit }: SlackSetupFormProps): JSX.Element => { - const form = useMemo(() => getSlackForm(), []); - - return ( -
- {({ Form, FormField, FormFooter }: FormProps) => ( - - - - {t`Finally, open Slack, create a public channel and enter its name below.`}{" "} - {t`This channel shouldn't really be used by anyone — we'll upload charts and tables here before sending out dashboard subscriptions (it's a Slack requirement).`} - - - - - )} - - ); + return ; }; export default SlackSetupForm; diff --git a/frontend/src/metabase/admin/settings/slack/components/SlackStatus/SlackStatus.stories.tsx b/frontend/src/metabase/admin/settings/slack/components/SlackStatus/SlackStatus.stories.tsx index b13f36a36634..a0013fa50296 100644 --- a/frontend/src/metabase/admin/settings/slack/components/SlackStatus/SlackStatus.stories.tsx +++ b/frontend/src/metabase/admin/settings/slack/components/SlackStatus/SlackStatus.stories.tsx @@ -1,5 +1,5 @@ import React from "react"; -import { ComponentStory } from "@storybook/react"; +import type { ComponentStory } from "@storybook/react"; import SlackStatus from "./SlackStatus"; export default { diff --git a/frontend/src/metabase/admin/settings/slack/components/SlackStatusForm/SlackStatusForm.tsx b/frontend/src/metabase/admin/settings/slack/components/SlackStatusForm/SlackStatusForm.tsx index 06bfaa908c03..a83ff003f48f 100644 --- a/frontend/src/metabase/admin/settings/slack/components/SlackStatusForm/SlackStatusForm.tsx +++ b/frontend/src/metabase/admin/settings/slack/components/SlackStatusForm/SlackStatusForm.tsx @@ -1,35 +1,13 @@ -import React, { useCallback, useMemo } from "react"; -import { t } from "ttag"; -import Form from "metabase/containers/FormikForm"; +import React from "react"; import { SlackSettings } from "metabase-types/api"; -import { getSlackForm } from "../../forms"; -import { FormProps } from "./types"; +import SlackForm from "../SlackForm"; export interface SlackStatusFormProps { settings: SlackSettings; } const SlackStatusForm = ({ settings }: SlackStatusFormProps): JSX.Element => { - const form = useMemo(() => getSlackForm(true), []); - const onSubmit = useCallback(() => undefined, []); - - return ( - - form={form} - initialValues={settings} - onSubmit={onSubmit} - > - {({ Form, FormField }: FormProps) => ( -
- - - - )} - - ); + return ; }; export default SlackStatusForm; diff --git a/frontend/src/metabase/admin/settings/slack/forms.ts b/frontend/src/metabase/admin/settings/slack/forms.ts deleted file mode 100644 index bed6642d25b4..000000000000 --- a/frontend/src/metabase/admin/settings/slack/forms.ts +++ /dev/null @@ -1,27 +0,0 @@ -import { t } from "ttag"; -import { SlackSettings } from "metabase-types/api"; -import { FormObject } from "metabase-types/forms"; - -export const getSlackForm = ( - readOnly?: boolean, -): FormObject => ({ - fields: [ - { - name: "slack-app-token", - type: "input", - title: t`Slack Bot User OAuth Token`, - placeholder: "xoxb-781236542736-2364535789652-GkwFDQoHqzXDVsC6GzqYUypD", - readOnly, - validate: (value: string) => !value && t`required`, - }, - { - name: "slack-files-channel", - type: "input", - title: t`Public channel to store image files`, - placeholder: "metabase_files", - readOnly, - validate: (value: string) => !value && t`required`, - normalize: (value: string) => value.toLowerCase(), - }, - ], -}); diff --git a/frontend/src/metabase/admin/settings/utils.js b/frontend/src/metabase/admin/settings/utils.js index cc30202dc3f9..a74c20710fa0 100644 --- a/frontend/src/metabase/admin/settings/utils.js +++ b/frontend/src/metabase/admin/settings/utils.js @@ -14,6 +14,7 @@ export const settingToFormField = setting => ({ ? t`Using ${setting.env_name}` : setting.placeholder || setting.default, validate: setting.required ? value => !value && "required" : undefined, + autoFocus: setting.autoFocus, }); export const settingToFormFieldId = setting => `setting-${setting.key}`; diff --git a/frontend/src/metabase/auth/actions.ts b/frontend/src/metabase/auth/actions.ts index 58678fd19e83..a8c99466b36e 100644 --- a/frontend/src/metabase/auth/actions.ts +++ b/frontend/src/metabase/auth/actions.ts @@ -97,13 +97,18 @@ export const resetPassword = createThunkAction( }, ); -export const VALIDATE_PASSWORD = "metabase/auth/VALIDATE_PASSWORD"; -export const validatePassword = createThunkAction( - VALIDATE_PASSWORD, - (password: string) => async () => { +export const validatePassword = async (password: string) => { + const error = MetabaseSettings.passwordComplexityDescription(password); + if (error) { + return error; + } + + try { await UtilApi.password_check({ password }); - }, -); + } catch (error) { + return getIn(error, ["data", "errors", "password"]); + } +}; export const VALIDATE_PASSWORD_TOKEN = "metabase/auth/VALIDATE_TOKEN"; export const validatePasswordToken = createThunkAction( diff --git a/frontend/src/metabase/auth/components/ForgotPassword/ForgotPassword.styled.tsx b/frontend/src/metabase/auth/components/ForgotPassword/ForgotPassword.styled.tsx index e488b261ab4b..2765084457a1 100644 --- a/frontend/src/metabase/auth/components/ForgotPassword/ForgotPassword.styled.tsx +++ b/frontend/src/metabase/auth/components/ForgotPassword/ForgotPassword.styled.tsx @@ -3,30 +3,6 @@ import { color } from "metabase/lib/colors"; import Icon from "metabase/components/Icon"; import Link from "metabase/core/components/Link"; -export const FormTitle = styled.div` - color: ${color("text-dark")}; - font-size: 1.25rem; - font-weight: 700; - line-height: 1.5rem; - text-align: center; - margin-bottom: 1.5rem; -`; - -export const FormFooter = styled.div` - display: flex; - flex-direction: column; - align-items: center; - margin-top: 1.5rem; -`; - -export const FormLink = styled(Link)` - color: ${color("text-dark")}; - - &:hover { - color: ${color("brand")}; - } -`; - export const InfoBody = styled.div` display: flex; flex-direction: column; diff --git a/frontend/src/metabase/auth/components/ForgotPassword/ForgotPassword.tsx b/frontend/src/metabase/auth/components/ForgotPassword/ForgotPassword.tsx index e0a700cb405a..ca88e16bbfbd 100644 --- a/frontend/src/metabase/auth/components/ForgotPassword/ForgotPassword.tsx +++ b/frontend/src/metabase/auth/components/ForgotPassword/ForgotPassword.tsx @@ -1,13 +1,9 @@ import React, { useCallback, useMemo, useState } from "react"; import { t } from "ttag"; -import Users from "metabase/entities/users"; import Button from "metabase/core/components/Button"; import AuthLayout from "../../containers/AuthLayout"; -import { ForgotPasswordData } from "../../types"; +import ForgotPasswordForm from "../ForgotPasswordForm"; import { - FormFooter, - FormLink, - FormTitle, InfoBody, InfoIcon, InfoIconContainer, @@ -54,43 +50,6 @@ const ForgotPassword = ({ ); }; -interface ForgotPasswordFormProps { - initialEmail?: string; - onSubmit: (email: string) => void; -} - -const ForgotPasswordForm = ({ - initialEmail, - onSubmit, -}: ForgotPasswordFormProps): JSX.Element => { - const initialValues = useMemo(() => { - return { email: initialEmail }; - }, [initialEmail]); - - const handleSubmit = useCallback( - async ({ email }: ForgotPasswordData) => { - await onSubmit(email); - }, - [onSubmit], - ); - - return ( -
- {t`Forgot password`} - - - {t`Back to sign in`} - -
- ); -}; - const ForgotPasswordSuccess = (): JSX.Element => { return ( diff --git a/frontend/src/metabase/auth/components/ForgotPassword/ForgotPassword.unit.spec.tsx b/frontend/src/metabase/auth/components/ForgotPassword/ForgotPassword.unit.spec.tsx index 4a96363e57a6..685dd6431c91 100644 --- a/frontend/src/metabase/auth/components/ForgotPassword/ForgotPassword.unit.spec.tsx +++ b/frontend/src/metabase/auth/components/ForgotPassword/ForgotPassword.unit.spec.tsx @@ -1,5 +1,5 @@ import React, { ReactNode } from "react"; -import { render, screen } from "@testing-library/react"; +import { render, screen, waitFor } from "@testing-library/react"; import userEvent from "@testing-library/user-event"; import ForgotPassword, { ForgotPasswordProps } from "./ForgotPassword"; @@ -13,16 +13,23 @@ describe("ForgotPassword", () => { }); it("should show a success message when the form is submitted", async () => { + const email = "user@metabase.test"; const props = getProps({ canResetPassword: true, onResetPassword: jest.fn().mockResolvedValue({}), }); render(); - userEvent.click(screen.getByText("Send password reset email")); + userEvent.type(screen.getByLabelText("Email address"), email); + await waitFor(() => { + expect(screen.getByText("Send password reset email")).toBeEnabled(); + }); - const message = await screen.findByText(/Check your email/); - expect(message).toBeInTheDocument(); + userEvent.click(screen.getByText("Send password reset email")); + await waitFor(() => { + expect(props.onResetPassword).toHaveBeenCalledWith(email); + expect(screen.getByText(/Check your email/)).toBeInTheDocument(); + }); }); it("should show an error message when the user cannot reset their password", () => { @@ -42,20 +49,6 @@ const getProps = ( ...opts, }); -interface FormMockProps { - submitTitle: string; - onSubmit: () => void; -} - -const FormMock = ({ submitTitle, onSubmit }: FormMockProps) => { - return ; -}; - -jest.mock("metabase/entities/users", () => ({ - forms: { password_reset: jest.fn() }, - Form: FormMock, -})); - interface AuthLayoutMockProps { children?: ReactNode; } diff --git a/frontend/src/metabase/auth/components/ForgotPasswordForm/ForgotPasswordForm.styled.tsx b/frontend/src/metabase/auth/components/ForgotPasswordForm/ForgotPasswordForm.styled.tsx new file mode 100644 index 000000000000..ffc6e887ac5d --- /dev/null +++ b/frontend/src/metabase/auth/components/ForgotPasswordForm/ForgotPasswordForm.styled.tsx @@ -0,0 +1,27 @@ +import styled from "@emotion/styled"; +import { color } from "metabase/lib/colors"; +import Link from "metabase/core/components/Link/Link"; + +export const PasswordFormTitle = styled.div` + color: ${color("text-dark")}; + font-size: 1.25rem; + font-weight: 700; + line-height: 1.5rem; + text-align: center; + margin-bottom: 1.5rem; +`; + +export const PasswordFormFooter = styled.div` + display: flex; + flex-direction: column; + align-items: center; + margin-top: 1.5rem; +`; + +export const PasswordFormLink = styled(Link)` + color: ${color("text-dark")}; + + &:hover { + color: ${color("brand")}; + } +`; diff --git a/frontend/src/metabase/auth/components/ForgotPasswordForm/ForgotPasswordForm.tsx b/frontend/src/metabase/auth/components/ForgotPasswordForm/ForgotPasswordForm.tsx new file mode 100644 index 000000000000..86444ce56cee --- /dev/null +++ b/frontend/src/metabase/auth/components/ForgotPasswordForm/ForgotPasswordForm.tsx @@ -0,0 +1,70 @@ +import React, { useCallback, useMemo } from "react"; +import { t } from "ttag"; +import * as Yup from "yup"; +import FormProvider from "metabase/core/components/FormProvider"; +import Form from "metabase/core/components/Form"; +import FormInput from "metabase/core/components/FormInput"; +import FormSubmitButton from "metabase/core/components/FormSubmitButton"; +import FormErrorMessage from "metabase/core/components/FormErrorMessage"; +import * as Errors from "metabase/core/utils/errors"; +import { ForgotPasswordData } from "../../types"; +import { + PasswordFormFooter, + PasswordFormLink, + PasswordFormTitle, +} from "./ForgotPasswordForm.styled"; + +const FORGOT_PASSWORD_SCHEMA = Yup.object({ + email: Yup.string().required(Errors.required).email(Errors.email), +}); + +export interface ForgotPasswordFormProps { + initialEmail?: string; + onSubmit: (email: string) => void; +} + +const ForgotPasswordForm = ({ + initialEmail = "", + onSubmit, +}: ForgotPasswordFormProps): JSX.Element => { + const initialValues = useMemo( + () => ({ email: initialEmail }), + [initialEmail], + ); + + const handleSubmit = useCallback( + ({ email }: ForgotPasswordData) => onSubmit(email), + [onSubmit], + ); + + return ( +
+ {t`Forgot password`} + +
+ + + + + + + {t`Back to sign in`} + +
+ ); +}; + +export default ForgotPasswordForm; diff --git a/frontend/src/metabase/auth/components/ForgotPasswordForm/index.ts b/frontend/src/metabase/auth/components/ForgotPasswordForm/index.ts new file mode 100644 index 000000000000..f56004f14883 --- /dev/null +++ b/frontend/src/metabase/auth/components/ForgotPasswordForm/index.ts @@ -0,0 +1 @@ +export { default } from "./ForgotPasswordForm"; diff --git a/frontend/src/metabase/auth/components/GoogleButton/GoogleButton.tsx b/frontend/src/metabase/auth/components/GoogleButton/GoogleButton.tsx index 41be42914f42..f5b5e4275724 100644 --- a/frontend/src/metabase/auth/components/GoogleButton/GoogleButton.tsx +++ b/frontend/src/metabase/auth/components/GoogleButton/GoogleButton.tsx @@ -1,7 +1,7 @@ import React, { useCallback, useState } from "react"; -import { GoogleOAuthProvider, GoogleLogin } from "@react-oauth/google"; import { t } from "ttag"; import { getIn } from "icepick"; +import { GoogleOAuthProvider, GoogleLogin } from "@react-oauth/google"; import * as Urls from "metabase/lib/urls"; import { AuthError, AuthErrorContainer, TextLink } from "./GoogleButton.styled"; diff --git a/frontend/src/metabase/auth/components/LoginForm/LoginForm.tsx b/frontend/src/metabase/auth/components/LoginForm/LoginForm.tsx index 7137907313fe..7869305ea604 100644 --- a/frontend/src/metabase/auth/components/LoginForm/LoginForm.tsx +++ b/frontend/src/metabase/auth/components/LoginForm/LoginForm.tsx @@ -1,25 +1,24 @@ -import React from "react"; +import React, { useMemo } from "react"; import { t } from "ttag"; -import { Form, Formik } from "formik"; import * as Yup from "yup"; -import useForm from "metabase/core/hooks/use-form"; +import Form from "metabase/core/components/Form"; +import FormProvider from "metabase/core/components/FormProvider"; import FormCheckBox from "metabase/core/components/FormCheckBox"; import FormErrorMessage from "metabase/core/components/FormErrorMessage"; -import FormField from "metabase/core/components/FormField"; import FormInput from "metabase/core/components/FormInput"; import FormSubmitButton from "metabase/core/components/FormSubmitButton"; +import * as Errors from "metabase/core/utils/errors"; import { LoginData } from "../../types"; -const LDAP_SCHEMA = Yup.object().shape({ - username: Yup.string().required(t`required`), - password: Yup.string().required(t`required`), - remember: Yup.boolean(), -}); - -const PASSWORD_SCHEMA = LDAP_SCHEMA.shape({ +const LOGIN_SCHEMA = Yup.object().shape({ username: Yup.string() - .required(t`required`) - .email(t`must be a valid email address`), + .required(Errors.required) + .when("$isLdapEnabled", { + is: false, + then: schema => schema.email(Errors.email), + }), + password: Yup.string().required(Errors.required), + remember: Yup.boolean(), }); export interface LoginFormProps { @@ -33,57 +32,52 @@ const LoginForm = ({ hasSessionCookies, onSubmit, }: LoginFormProps): JSX.Element => { - const initialValues: LoginData = { - username: "", - password: "", - remember: !hasSessionCookies, - }; - const handleSubmit = useForm(onSubmit); + const initialValues = useMemo( + () => ({ + username: "", + password: "", + remember: !hasSessionCookies, + }), + [hasSessionCookies], + ); + + const validationContext = useMemo( + () => ({ + isLdapEnabled, + }), + [isLdapEnabled], + ); return ( -
- - - - - - + type={isLdapEnabled ? "input" : "email"} + placeholder="nicetoseeyou@email.com" + autoFocus + /> + {!hasSessionCookies && ( - - - + )} - + - + ); }; diff --git a/frontend/src/metabase/auth/components/ResetPassword/ResetPassword.styled.tsx b/frontend/src/metabase/auth/components/ResetPassword/ResetPassword.styled.tsx index 53c4971e33ff..c036766b4a8f 100644 --- a/frontend/src/metabase/auth/components/ResetPassword/ResetPassword.styled.tsx +++ b/frontend/src/metabase/auth/components/ResetPassword/ResetPassword.styled.tsx @@ -1,21 +1,5 @@ import styled from "@emotion/styled"; import { color } from "metabase/lib/colors"; -import Icon from "metabase/components/Icon"; - -export const FormTitle = styled.div` - color: ${color("text-dark")}; - font-size: 1.25rem; - font-weight: 700; - line-height: 1.5rem; - text-align: center; - margin-bottom: 1rem; -`; - -export const FormMessage = styled.div` - color: ${color("text-dark")}; - text-align: center; - margin-bottom: 1.5rem; -`; export const InfoBody = styled.div` display: flex; diff --git a/frontend/src/metabase/auth/components/ResetPassword/ResetPassword.tsx b/frontend/src/metabase/auth/components/ResetPassword/ResetPassword.tsx index 5fb0b8786abd..1b0623507136 100644 --- a/frontend/src/metabase/auth/components/ResetPassword/ResetPassword.tsx +++ b/frontend/src/metabase/auth/components/ResetPassword/ResetPassword.tsx @@ -1,25 +1,18 @@ -import React, { useCallback, useEffect, useMemo, useState } from "react"; +import React, { useCallback, useEffect, useState } from "react"; import { t } from "ttag"; -import { getIn } from "icepick"; -import Settings from "metabase/lib/settings"; -import Users from "metabase/entities/users"; +import Button from "metabase/core/components/Button"; import Link from "metabase/core/components/Link"; import AuthLayout from "../../containers/AuthLayout"; +import ResetPasswordForm from "../ResetPasswordForm"; import { ResetPasswordData } from "../../types"; -import { - FormMessage, - FormTitle, - InfoBody, - InfoMessage, - InfoTitle, -} from "./ResetPassword.styled"; +import { InfoBody, InfoMessage, InfoTitle } from "./ResetPassword.styled"; type ViewType = "none" | "form" | "success" | "expired"; export interface ResetPasswordProps { token: string; onResetPassword: (token: string, password: string) => void; - onValidatePassword: (password: string) => void; + onValidatePassword: (password: string) => Promise; onValidatePasswordToken: (token: string) => void; onShowToast: (toast: { message: string }) => void; onRedirect: (url: string) => void; @@ -44,18 +37,6 @@ const ResetPassword = ({ } }, [token, onValidatePasswordToken]); - const handlePasswordChange = useCallback( - async ({ password }: ResetPasswordData) => { - try { - await onValidatePassword(password); - return {}; - } catch (error) { - return getPasswordError(error); - } - }, - [onValidatePassword], - ); - const handlePasswordSubmit = useCallback( async ({ password }: ResetPasswordData) => { await onResetPassword(token, password); @@ -73,7 +54,7 @@ const ResetPassword = ({ {view === "form" && ( )} @@ -82,36 +63,6 @@ const ResetPassword = ({ ); }; -interface ResetPasswordFormProps { - onPasswordChange: (data: ResetPasswordData) => void; - onSubmit: (data: ResetPasswordData) => void; -} - -const ResetPasswordForm = ({ - onPasswordChange, - onSubmit, -}: ResetPasswordFormProps): JSX.Element => { - const passwordDescription = useMemo( - () => Settings.passwordComplexityDescription(), - [], - ); - - return ( -
- {t`New password`} - {t`To keep your data secure, passwords ${passwordDescription}`} - -
- ); -}; - const ResetPasswordExpired = (): JSX.Element => { return ( @@ -119,16 +70,11 @@ const ResetPasswordExpired = (): JSX.Element => { {t`For security reasons, password reset links expire after a little while. If you still need to reset your password, you can request a new reset email.`} - {t`Request a new reset email`} + ); }; -const getPasswordError = (error: unknown) => { - return getIn(error, ["data", "errors"]); -}; - export default ResetPassword; diff --git a/frontend/src/metabase/auth/components/ResetPassword/ResetPassword.unit.spec.tsx b/frontend/src/metabase/auth/components/ResetPassword/ResetPassword.unit.spec.tsx index 833d2257ef5b..040aaeb92943 100644 --- a/frontend/src/metabase/auth/components/ResetPassword/ResetPassword.unit.spec.tsx +++ b/frontend/src/metabase/auth/components/ResetPassword/ResetPassword.unit.spec.tsx @@ -11,8 +11,10 @@ describe("ResetPassword", () => { render(); - const message = await screen.findByText("New password"); - expect(message).toBeInTheDocument(); + await waitFor(() => { + expect(props.onValidatePasswordToken).toHaveBeenCalledWith(props.token); + expect(screen.getByText("New password")).toBeInTheDocument(); + }); }); it("should show an error message when token validation fails", async () => { @@ -22,36 +24,40 @@ describe("ResetPassword", () => { render(); - const message = await screen.findByText("Whoops, that's an expired link"); - expect(message).toBeInTheDocument(); + await waitFor(() => { + expect(props.onValidatePasswordToken).toHaveBeenCalledWith(props.token); + expect(screen.getByText(/that's an expired link/)).toBeInTheDocument(); + }); }); it("should show a success message when the form is submitted", async () => { - const onShowToast = jest.fn(); - const onRedirect = jest.fn(); - const props = getProps({ onResetPassword: jest.fn().mockResolvedValue({}), + onValidatePassword: jest.fn().mockResolvedValue(undefined), onValidatePasswordToken: jest.fn().mockResolvedValue({}), - onShowToast, - onRedirect, }); - render( - , - ); + render(); - const button = await screen.findByText("Save new password"); + await waitFor(() => { + expect(props.onValidatePasswordToken).toHaveBeenCalledWith(props.token); + expect(screen.getByText("New password")).toBeInTheDocument(); + }); - userEvent.click(button); + userEvent.type(screen.getByLabelText("Create a password"), "test"); + userEvent.type(screen.getByLabelText("Confirm your password"), "test"); await waitFor(() => { - expect(onRedirect).toHaveBeenCalledWith("/"); - expect(onShowToast).toHaveBeenCalledWith({ + expect(props.onValidatePassword).toHaveBeenCalledWith("test"); + expect(screen.getByText("Save new password")).toBeEnabled(); + }); + + userEvent.click(screen.getByText("Save new password")); + + await waitFor(() => { + expect(props.onResetPassword).toHaveBeenCalledWith(props.token, "test"); + expect(props.onRedirect).toHaveBeenCalledWith("/"); + expect(props.onShowToast).toHaveBeenCalledWith({ message: "You've updated your password.", }); }); @@ -70,20 +76,6 @@ const getProps = (opts?: Partial): ResetPasswordProps => { }; }; -interface FormMockProps { - submitTitle: string; - onSubmit: () => void; -} - -const FormMock = ({ submitTitle, onSubmit }: FormMockProps) => { - return ; -}; - -jest.mock("metabase/entities/users", () => ({ - forms: { password_reset: jest.fn() }, - Form: FormMock, -})); - interface AuthLayoutMockProps { children?: ReactNode; } diff --git a/frontend/src/metabase/auth/components/ResetPasswordForm/ResetPasswordForm.styled.tsx b/frontend/src/metabase/auth/components/ResetPasswordForm/ResetPasswordForm.styled.tsx new file mode 100644 index 000000000000..7f150701de95 --- /dev/null +++ b/frontend/src/metabase/auth/components/ResetPasswordForm/ResetPasswordForm.styled.tsx @@ -0,0 +1,17 @@ +import styled from "@emotion/styled"; +import { color } from "metabase/lib/colors"; + +export const PasswordFormTitle = styled.div` + color: ${color("text-dark")}; + font-size: 1.25rem; + font-weight: 700; + line-height: 1.5rem; + text-align: center; + margin-bottom: 1rem; +`; + +export const PasswordFormMessage = styled.div` + color: ${color("text-dark")}; + text-align: center; + margin-bottom: 1.5rem; +`; diff --git a/frontend/src/metabase/auth/components/ResetPasswordForm/ResetPasswordForm.tsx b/frontend/src/metabase/auth/components/ResetPasswordForm/ResetPasswordForm.tsx new file mode 100644 index 000000000000..25794d61bd61 --- /dev/null +++ b/frontend/src/metabase/auth/components/ResetPasswordForm/ResetPasswordForm.tsx @@ -0,0 +1,90 @@ +import React, { useMemo } from "react"; +import { t } from "ttag"; +import * as Yup from "yup"; +import _ from "underscore"; +import MetabaseSettings from "metabase/lib/settings"; +import Form from "metabase/core/components/Form"; +import FormProvider from "metabase/core/components/FormProvider"; +import FormInput from "metabase/core/components/FormInput"; +import FormSubmitButton from "metabase/core/components/FormSubmitButton"; +import FormErrorMessage from "metabase/core/components/FormErrorMessage"; +import * as Errors from "metabase/core/utils/errors"; +import { ResetPasswordData } from "../../types"; +import { + PasswordFormMessage, + PasswordFormTitle, +} from "./ResetPasswordForm.styled"; + +const RESET_PASSWORD_SCHEMA = Yup.object({ + password: Yup.string() + .default("") + .required(Errors.required) + .test(async (value = "", context) => { + const error = await context.options.context?.onValidatePassword(value); + return error ? context.createError({ message: error }) : true; + }), + password_confirm: Yup.string() + .default("") + .required(Errors.required) + .oneOf([Yup.ref("password")], t`passwords do not match`), +}); + +interface ResetPasswordFormProps { + onValidatePassword: (password: string) => Promise; + onSubmit: (data: ResetPasswordData) => void; +} + +const ResetPasswordForm = ({ + onValidatePassword, + onSubmit, +}: ResetPasswordFormProps): JSX.Element => { + const initialValues = useMemo(() => { + return RESET_PASSWORD_SCHEMA.getDefault(); + }, []); + + const passwordDescription = useMemo(() => { + return MetabaseSettings.passwordComplexityDescription(); + }, []); + + const validationContext = useMemo( + () => ({ onValidatePassword: _.memoize(onValidatePassword) }), + [onValidatePassword], + ); + + return ( +
+ {t`New password`} + + {t`To keep your data secure, passwords ${passwordDescription}`} + + +
+ + + + + + +
+ ); +}; + +export default ResetPasswordForm; diff --git a/frontend/src/metabase/auth/components/ResetPasswordForm/index.ts b/frontend/src/metabase/auth/components/ResetPasswordForm/index.ts new file mode 100644 index 000000000000..21ff6c5a72af --- /dev/null +++ b/frontend/src/metabase/auth/components/ResetPasswordForm/index.ts @@ -0,0 +1 @@ +export { default } from "./ResetPasswordForm"; diff --git a/frontend/src/metabase/auth/containers/ResetPasswordApp/ResetPasswordApp.tsx b/frontend/src/metabase/auth/containers/ResetPasswordApp/ResetPasswordApp.tsx index 5ee05f253a42..7b721650d14a 100644 --- a/frontend/src/metabase/auth/containers/ResetPasswordApp/ResetPasswordApp.tsx +++ b/frontend/src/metabase/auth/containers/ResetPasswordApp/ResetPasswordApp.tsx @@ -10,11 +10,11 @@ import { const mapStateToProps = (state: any, props: any) => ({ token: props.params.token, + onValidatePassword: validatePassword, }); const mapDispatchToProps = { onResetPassword: resetPassword, - onValidatePassword: validatePassword, onValidatePasswordToken: validatePasswordToken, onShowToast: addUndo, onRedirect: replace, diff --git a/frontend/src/metabase/collections/components/CollectionCopyEntityModal.jsx b/frontend/src/metabase/collections/components/CollectionCopyEntityModal.jsx index 91c1fbe9aaae..fac29a618ad4 100644 --- a/frontend/src/metabase/collections/components/CollectionCopyEntityModal.jsx +++ b/frontend/src/metabase/collections/components/CollectionCopyEntityModal.jsx @@ -1,5 +1,5 @@ /* eslint-disable react/prop-types */ -import React from "react"; +import React, { useState } from "react"; import { connect } from "react-redux"; import { dissoc } from "icepick"; import { t } from "ttag"; @@ -10,6 +10,7 @@ import { entityTypeForObject } from "metabase/lib/schema"; import Link from "metabase/core/components/Link"; +import Dashboards from "metabase/entities/dashboards"; import Collections from "metabase/entities/collections"; import EntityCopyModal from "metabase/entities/containers/EntityCopyModal"; @@ -22,6 +23,16 @@ function mapStateToProps(state, props) { }; } +const getTitle = (entityObject, isShallowCopy) => { + if (entityObject.model !== "dashboard") { + return ""; + } else if (isShallowCopy) { + return t`Duplicate "${entityObject.name}"`; + } else { + return t`Duplicate "${entityObject.name}" and its questions`; + } +}; + function CollectionCopyEntityModal({ entityObject, initialCollectionId, @@ -29,6 +40,35 @@ function CollectionCopyEntityModal({ onSaved, triggerToast, }) { + const [isShallowCopy, setIsShallowCopy] = useState(true); + const title = getTitle(entityObject, isShallowCopy); + + const handleValuesChange = ({ is_shallow_copy }) => { + setIsShallowCopy(is_shallow_copy); + }; + + const handleSaved = newEntityObject => { + const newEntityUrl = Urls.modelToUrl({ + model: entityObject.model, + model_object: newEntityObject, + }); + + triggerToast( +
+ {/* A shallow-copied newEntityObject will not include `uncopied` */} + {newEntityObject.uncopied?.length > 0 + ? t`Duplicated ${entityObject.model}, but couldn't duplicate some questions` + : t`Duplicated ${entityObject.model}`} + + {t`See it`} + +
, + { icon: entityObject.model }, + ); + + onSaved(newEntityObject); + }; + return ( { return entityObject.copy(dissoc(values, "id")); }} onClose={onClose} - onSaved={newEntityObject => { - const newEntityUrl = Urls.modelToUrl({ - model: entityObject.model, - model_object: newEntityObject, - }); - triggerToast( -
- {t`Duplicated ${entityObject.model}`} - - {t`See it`} - -
, - { icon: entityObject.model }, - ); - - onSaved(newEntityObject); - }} + onSaved={handleSaved} + onValuesChange={handleValuesChange} /> ); } diff --git a/frontend/src/metabase/collections/components/PinnedItemCard/PinnedItemCard.stories.tsx b/frontend/src/metabase/collections/components/PinnedItemCard/PinnedItemCard.stories.tsx index 98a93ef6c736..49dc6aefcb71 100644 --- a/frontend/src/metabase/collections/components/PinnedItemCard/PinnedItemCard.stories.tsx +++ b/frontend/src/metabase/collections/components/PinnedItemCard/PinnedItemCard.stories.tsx @@ -1,7 +1,7 @@ import React from "react"; import { action } from "@storybook/addon-actions"; -import { ComponentStory } from "@storybook/react"; +import type { ComponentStory } from "@storybook/react"; import PinnedItemCard from "./PinnedItemCard"; export default { diff --git a/frontend/src/metabase/components/Calendar.stories.tsx b/frontend/src/metabase/components/Calendar.stories.tsx index 538bdd1ce030..55b0569d6e2b 100644 --- a/frontend/src/metabase/components/Calendar.stories.tsx +++ b/frontend/src/metabase/components/Calendar.stories.tsx @@ -1,5 +1,5 @@ import React from "react"; -import { ComponentStory } from "@storybook/react"; +import type { ComponentStory } from "@storybook/react"; import Calendar from "./Calendar"; export default { diff --git a/frontend/src/metabase/components/DatabaseHelpCard/DatabaseHelpCard.stories.tsx b/frontend/src/metabase/components/DatabaseHelpCard/DatabaseHelpCard.stories.tsx index d10a28091360..457851cb774d 100644 --- a/frontend/src/metabase/components/DatabaseHelpCard/DatabaseHelpCard.stories.tsx +++ b/frontend/src/metabase/components/DatabaseHelpCard/DatabaseHelpCard.stories.tsx @@ -1,5 +1,5 @@ import React from "react"; -import { ComponentStory } from "@storybook/react"; +import type { ComponentStory } from "@storybook/react"; import DatabaseHelpCard from "./DatabaseHelpCard"; export default { diff --git a/frontend/src/metabase/components/DateMonthYearWidget/DateMonthYearWidget.stories.tsx b/frontend/src/metabase/components/DateMonthYearWidget/DateMonthYearWidget.stories.tsx index cb1c2ffda4db..fc76f5c56b0a 100644 --- a/frontend/src/metabase/components/DateMonthYearWidget/DateMonthYearWidget.stories.tsx +++ b/frontend/src/metabase/components/DateMonthYearWidget/DateMonthYearWidget.stories.tsx @@ -1,5 +1,5 @@ import React from "react"; -import { ComponentStory } from "@storybook/react"; +import type { ComponentStory } from "@storybook/react"; import { useArgs } from "@storybook/client-api"; import DateMonthYearWidget from "./DateMonthYearWidget"; diff --git a/frontend/src/metabase/components/DateQuarterYearWidget/DateQuarterYearWidget.stories.tsx b/frontend/src/metabase/components/DateQuarterYearWidget/DateQuarterYearWidget.stories.tsx index c997e0259780..c6901586138a 100644 --- a/frontend/src/metabase/components/DateQuarterYearWidget/DateQuarterYearWidget.stories.tsx +++ b/frontend/src/metabase/components/DateQuarterYearWidget/DateQuarterYearWidget.stories.tsx @@ -1,5 +1,5 @@ import React from "react"; -import { ComponentStory } from "@storybook/react"; +import type { ComponentStory } from "@storybook/react"; import { useArgs } from "@storybook/client-api"; import DateQuarterYearWidget from "./DateQuarterYearWidget"; diff --git a/frontend/src/metabase/components/DateRelativeWidget/DateRelativeWidget.stories.tsx b/frontend/src/metabase/components/DateRelativeWidget/DateRelativeWidget.stories.tsx index bc7b5767dbb8..10557371ad94 100644 --- a/frontend/src/metabase/components/DateRelativeWidget/DateRelativeWidget.stories.tsx +++ b/frontend/src/metabase/components/DateRelativeWidget/DateRelativeWidget.stories.tsx @@ -1,5 +1,5 @@ import React from "react"; -import { ComponentStory } from "@storybook/react"; +import type { ComponentStory } from "@storybook/react"; import { useArgs } from "@storybook/client-api"; import DateRelativeWidget from "./DateRelativeWidget"; diff --git a/frontend/src/metabase/components/DriverWarning/DriverWarning.stories.tsx b/frontend/src/metabase/components/DriverWarning/DriverWarning.stories.tsx index 3726460a8233..d13f940693b8 100644 --- a/frontend/src/metabase/components/DriverWarning/DriverWarning.stories.tsx +++ b/frontend/src/metabase/components/DriverWarning/DriverWarning.stories.tsx @@ -1,5 +1,5 @@ import React from "react"; -import { ComponentStory } from "@storybook/react"; +import type { ComponentStory } from "@storybook/react"; import { createMockEngine } from "metabase-types/api/mocks"; import DriverWarning from "./DriverWarning"; diff --git a/frontend/src/metabase/components/FieldValuesWidget.jsx b/frontend/src/metabase/components/FieldValuesWidget.jsx index afd601a068f9..4031d24d5f93 100644 --- a/frontend/src/metabase/components/FieldValuesWidget.jsx +++ b/frontend/src/metabase/components/FieldValuesWidget.jsx @@ -23,6 +23,11 @@ import { stripId } from "metabase/lib/formatting"; import { fetchDashboardParameterValues } from "metabase/dashboard/actions"; import Fields from "metabase/entities/fields"; +import { + isIdParameter, + isNumberParameter, + isStringParameter, +} from "metabase-lib/parameters/utils/parameter-type"; const MAX_SEARCH_RESULTS = 100; @@ -291,6 +296,7 @@ class FieldValuesWidgetInner extends Component { const tokenFieldPlaceholder = getTokenFieldPlaceholder({ fields, + parameter, disableSearch, placeholder, disablePKRemappingForSearch, @@ -307,6 +313,12 @@ class FieldValuesWidgetInner extends Component { const isLoading = loadingState === "LOADING"; const hasListValues = hasList({ fields, disableSearch, options }); + const parseFreeformValue = value => { + return isNumeric(fields[0], parameter) + ? parseNumberValue(value) + : parseStringValue(value); + }; + return (
{ - return fields[0].isNumeric() - ? parseNumberValue(value) - : parseStringValue(value); - }} + parseFreeformValue={parseFreeformValue} /> )}
@@ -446,17 +454,30 @@ function shouldList(fields, disableSearch) { ); } -function getNonSearchableTokenFieldPlaceholder(firstField) { - if (firstField.isID()) { - return t`Enter an ID`; - } else if (firstField.isString()) { +function getNonSearchableTokenFieldPlaceholder(firstField, parameter) { + if (parameter) { + if (isIdParameter(parameter)) { + return t`Enter an ID`; + } else if (isStringParameter(parameter)) { + return t`Enter some text`; + } else if (isNumberParameter(parameter)) { + return t`Enter a number`; + } + + // fallback return t`Enter some text`; - } else if (firstField.isNumeric()) { - return t`Enter a number`; - } + } else { + if (firstField.isID()) { + return t`Enter an ID`; + } else if (firstField.isString()) { + return t`Enter some text`; + } else if (firstField.isNumeric()) { + return t`Enter a number`; + } - // fallback - return t`Enter some text`; + // fallback + return t`Enter some text`; + } } export function searchField(field, disablePKRemappingForSearch) { @@ -544,6 +565,7 @@ export function isSearchable({ function getTokenFieldPlaceholder({ fields, + parameter, disableSearch, placeholder, disablePKRemappingForSearch, @@ -579,7 +601,7 @@ function getTokenFieldPlaceholder({ disablePKRemappingForSearch, ); } else { - return getNonSearchableTokenFieldPlaceholder(firstField); + return getNonSearchableTokenFieldPlaceholder(firstField, parameter); } } @@ -672,3 +694,11 @@ export function getValuesMode( return "none"; } + +function isNumeric(field, parameter) { + if (parameter) { + return isNumberParameter(parameter); + } + + return field.isNumeric(); +} diff --git a/frontend/src/metabase/components/HelpCard/HelpCard.stories.tsx b/frontend/src/metabase/components/HelpCard/HelpCard.stories.tsx index 889bdaf388a6..8d344fa4e343 100644 --- a/frontend/src/metabase/components/HelpCard/HelpCard.stories.tsx +++ b/frontend/src/metabase/components/HelpCard/HelpCard.stories.tsx @@ -1,5 +1,5 @@ import React from "react"; -import { ComponentStory } from "@storybook/react"; +import type { ComponentStory } from "@storybook/react"; import HelpCard from "./HelpCard"; export default { diff --git a/frontend/src/metabase/components/InputBlurChange.jsx b/frontend/src/metabase/components/InputBlurChange.jsx index fb3ee79421d2..e97dc1ee5aab 100644 --- a/frontend/src/metabase/components/InputBlurChange.jsx +++ b/frontend/src/metabase/components/InputBlurChange.jsx @@ -33,7 +33,9 @@ export default class InputBlurChange extends Component { }; UNSAFE_componentWillReceiveProps(newProps) { - this.setState({ value: newProps.value }); + if (newProps.value !== this.state.value) { + this.setState({ value: newProps.value }); + } } onChange(event) { diff --git a/frontend/src/metabase/components/InputBlurChange.styled.tsx b/frontend/src/metabase/components/InputBlurChange.styled.tsx index b23ded2c2341..13a04c387116 100644 --- a/frontend/src/metabase/components/InputBlurChange.styled.tsx +++ b/frontend/src/metabase/components/InputBlurChange.styled.tsx @@ -1,16 +1,21 @@ import styled from "@emotion/styled"; +import { + inputFocusOutline, + inputPadding, + inputTypography, + numericInputReset, +} from "metabase/core/style/input"; import { color } from "metabase/lib/colors"; export const Input = styled.input` - border: 1px solid ${color("border")}; + ${inputPadding}; + ${inputTypography}; + border: 1px solid ${() => color("border")}; + border-radius: 0.5rem; + color: ${() => color("text-dark")}; + transition: border 0.3s; - &::-webkit-outer-spin-button, - &::-webkit-inner-spin-button { - -webkit-appearance: none; - margin: 0; - } + ${inputFocusOutline}; - &[type="number"] { - -moz-appearance: textfield; - } + ${numericInputReset}; `; diff --git a/frontend/src/metabase/components/SegmentedControl.styled.jsx b/frontend/src/metabase/components/SegmentedControl.styled.jsx index 2afb5f342030..49727599cb2e 100644 --- a/frontend/src/metabase/components/SegmentedControl.styled.jsx +++ b/frontend/src/metabase/components/SegmentedControl.styled.jsx @@ -19,8 +19,7 @@ const COLORS = { "fill-background": { background: ({ isSelected, selectedColor }) => isSelected ? color(selectedColor) : "transparent", - border: ({ isSelected, selectedColor }) => - isSelected ? color(selectedColor) : getDefaultBorderColor(), + border: ({ selectedColor }) => color(selectedColor), text: ({ isSelected, inactiveColor }) => color(isSelected ? "text-white" : inactiveColor), }, diff --git a/frontend/src/metabase/components/SelectList/SelectListItem.styled.jsx b/frontend/src/metabase/components/SelectList/SelectListItem.styled.jsx index 03286481f6c0..11c898092e35 100644 --- a/frontend/src/metabase/components/SelectList/SelectListItem.styled.jsx +++ b/frontend/src/metabase/components/SelectList/SelectListItem.styled.jsx @@ -2,7 +2,7 @@ import styled from "@emotion/styled"; import { css } from "@emotion/react"; import Label from "metabase/components/type/Label"; -import { color, lighten } from "metabase/lib/colors"; +import { color } from "metabase/lib/colors"; import Icon from "metabase/components/Icon"; export const ItemTitle = styled(Label)` @@ -16,11 +16,11 @@ export const ItemIcon = styled(Icon)` `; const activeItemCss = css` - background-color: ${lighten("brand")}; + background-color: ${color("brand")}; ${ItemIcon}, ${ItemTitle} { - color: ${color("brand")}; + color: ${color("white")}; } `; diff --git a/frontend/src/metabase/components/TextWidget/TextWidget.stories.tsx b/frontend/src/metabase/components/TextWidget/TextWidget.stories.tsx index 664ce2825c85..41b347d8eca3 100644 --- a/frontend/src/metabase/components/TextWidget/TextWidget.stories.tsx +++ b/frontend/src/metabase/components/TextWidget/TextWidget.stories.tsx @@ -1,5 +1,5 @@ import React from "react"; -import { ComponentStory } from "@storybook/react"; +import type { ComponentStory } from "@storybook/react"; import { useArgs } from "@storybook/client-api"; import TextWidget from "./TextWidget"; @@ -11,7 +11,7 @@ export default { const Template: ComponentStory = args => { const [{ value }, updateArgs] = useArgs(); - const setValue = (value: string | null) => { + const setValue = (value: string | number | null) => { updateArgs({ value }); }; diff --git a/frontend/src/metabase/components/TextWidget/TextWidget.tsx b/frontend/src/metabase/components/TextWidget/TextWidget.tsx index f0209a05620e..232fb3e4e8bc 100644 --- a/frontend/src/metabase/components/TextWidget/TextWidget.tsx +++ b/frontend/src/metabase/components/TextWidget/TextWidget.tsx @@ -5,8 +5,8 @@ import { forceRedraw } from "metabase/lib/dom"; import { KEYCODE_ENTER, KEYCODE_ESCAPE } from "metabase/lib/keyboard"; type Props = { - value: string; - setValue: (v: string | null) => void; + value: string | number; + setValue: (v: string | number | null) => void; className?: string; isEditing: boolean; commitImmediately?: boolean; @@ -16,7 +16,7 @@ type Props = { }; type State = { - value: string | null; + value: string | number | null; isFocused: boolean; }; @@ -73,11 +73,11 @@ class TextWidget extends React.Component { { this.setState({ value: e.target.value }); if (this.props.commitImmediately) { - this.props.setValue(e.target.value || null); + this.props.setValue(e.target.value ?? null); } }} onKeyUp={e => { @@ -85,7 +85,7 @@ class TextWidget extends React.Component { if (e.keyCode === KEYCODE_ESCAPE) { target.blur(); } else if (e.keyCode === KEYCODE_ENTER) { - setValue(this.state.value || null); + setValue(this.state.value ?? null); target.blur(); } }} diff --git a/frontend/src/metabase/components/TextWidget/TextWidget.unit.spec.tsx b/frontend/src/metabase/components/TextWidget/TextWidget.unit.spec.tsx index 1072adb540a1..76fcd823acb8 100644 --- a/frontend/src/metabase/components/TextWidget/TextWidget.unit.spec.tsx +++ b/frontend/src/metabase/components/TextWidget/TextWidget.unit.spec.tsx @@ -1,7 +1,16 @@ -import React from "react"; +import React, { useState } from "react"; import { fireEvent, render, screen } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { act } from "react-dom/test-utils"; import TextWidget from "./TextWidget"; +const TextInputWithStateWrapper = ({ value }: { value?: number | string }) => { + const [val, setVal] = useState(value ?? ""); + return ( + + ); +}; + describe("TextWidget", () => { it("should render correctly", () => { render( @@ -28,4 +37,30 @@ describe("TextWidget", () => { }); expect(screen.getByRole("textbox")).toHaveValue("Toucan McBird"); }); + + it("should render a zero as an initial value", () => { + render( + , + ); + + expect(screen.getByRole("textbox")).toHaveValue("0"); + }); + + it("should accept zero as an input value", async () => { + render(); + + const textbox = screen.getByRole("textbox"); + + await userEvent.type(textbox, "0"); + expect(textbox).toHaveValue("0"); + }); + + it("should keep zero value when pressing enter", async () => { + render(); + + const textbox = screen.getByRole("textbox"); + + await userEvent.type(textbox, "0{enter}"); + expect(textbox).toHaveValue("0"); + }); }); diff --git a/frontend/src/metabase/components/Toaster/Toaster.stories.tsx b/frontend/src/metabase/components/Toaster/Toaster.stories.tsx index 2accc0ce798c..e4d65a6053d7 100644 --- a/frontend/src/metabase/components/Toaster/Toaster.stories.tsx +++ b/frontend/src/metabase/components/Toaster/Toaster.stories.tsx @@ -1,5 +1,5 @@ import React from "react"; -import { ComponentStory } from "@storybook/react"; +import type { ComponentStory } from "@storybook/react"; import Toaster from "./Toaster"; export default { diff --git a/frontend/src/metabase/components/TokenFieldItem/TokenFieldItem.stories.tsx b/frontend/src/metabase/components/TokenFieldItem/TokenFieldItem.stories.tsx index a762b6ad1508..cafaa6103971 100644 --- a/frontend/src/metabase/components/TokenFieldItem/TokenFieldItem.stories.tsx +++ b/frontend/src/metabase/components/TokenFieldItem/TokenFieldItem.stories.tsx @@ -1,5 +1,5 @@ import React from "react"; -import { ComponentStory } from "@storybook/react"; +import type { ComponentStory } from "@storybook/react"; import Icon from "../Icon"; import { TokenFieldItem, TokenFieldAddon } from "./TokenFieldItem.styled"; diff --git a/frontend/src/metabase/components/UserAvatar/UserAvartar.stories.tsx b/frontend/src/metabase/components/UserAvatar/UserAvartar.stories.tsx index 111340197046..f73b96345819 100644 --- a/frontend/src/metabase/components/UserAvatar/UserAvartar.stories.tsx +++ b/frontend/src/metabase/components/UserAvatar/UserAvartar.stories.tsx @@ -1,5 +1,5 @@ import React from "react"; -import { ComponentStory } from "@storybook/react"; +import type { ComponentStory } from "@storybook/react"; import UserAvatar from "./UserAvatar"; export default { diff --git a/frontend/src/metabase/components/YearPicker/YearPicker.stories.tsx b/frontend/src/metabase/components/YearPicker/YearPicker.stories.tsx index 77824720b5b1..ceb1062d696d 100644 --- a/frontend/src/metabase/components/YearPicker/YearPicker.stories.tsx +++ b/frontend/src/metabase/components/YearPicker/YearPicker.stories.tsx @@ -1,5 +1,5 @@ import React from "react"; -import { ComponentStory } from "@storybook/react"; +import type { ComponentStory } from "@storybook/react"; import { useArgs } from "@storybook/client-api"; import YearPicker from "./YearPicker"; diff --git a/frontend/src/metabase/components/form/CustomForm/CustomForm.tsx b/frontend/src/metabase/components/form/CustomForm/CustomForm.tsx index e4fb33234bae..a6e61319a4e7 100644 --- a/frontend/src/metabase/components/form/CustomForm/CustomForm.tsx +++ b/frontend/src/metabase/components/form/CustomForm/CustomForm.tsx @@ -50,6 +50,9 @@ function CustomForm(props: CustomFormProps) { return
; } +/** + * @deprecated + */ class CustomFormWithLegacyContext extends React.Component { static childContextTypes = LegacyContextTypes; diff --git a/frontend/src/metabase/components/form/CustomForm/CustomFormField.tsx b/frontend/src/metabase/components/form/CustomForm/CustomFormField.tsx index 9dbcccddd818..ee0f6f49f5b6 100644 --- a/frontend/src/metabase/components/form/CustomForm/CustomFormField.tsx +++ b/frontend/src/metabase/components/form/CustomForm/CustomFormField.tsx @@ -130,8 +130,14 @@ CustomFormFieldLegacyContext.contextTypes = { unregisterFormField: PropTypes.func, }; -export default React.forwardRef( - function CustomFormField(props, ref) { - return ; - }, -); +/** + * @deprecated + */ +const CustomFormField = React.forwardRef< + HTMLInputElement, + CustomFormFieldProps +>(function CustomFormField(props, ref) { + return ; +}); + +export default CustomFormField; diff --git a/frontend/src/metabase/components/form/CustomForm/CustomFormFooter/CustomFormFooter.tsx b/frontend/src/metabase/components/form/CustomForm/CustomFormFooter/CustomFormFooter.tsx index c3bb119b823d..fb0f108256fc 100644 --- a/frontend/src/metabase/components/form/CustomForm/CustomFormFooter/CustomFormFooter.tsx +++ b/frontend/src/metabase/components/form/CustomForm/CustomFormFooter/CustomFormFooter.tsx @@ -37,6 +37,9 @@ function CustomFormFooter({ ); } +/** + * @deprecated + */ const CustomFormFooterLegacyContext = ( props: CustomFormFooterProps, { isModal: isContextModal }: LegacyContextProps, diff --git a/frontend/src/metabase/components/form/CustomForm/CustomFormMessage.tsx b/frontend/src/metabase/components/form/CustomForm/CustomFormMessage.tsx index 381d4376eb6a..b293875b8376 100644 --- a/frontend/src/metabase/components/form/CustomForm/CustomFormMessage.tsx +++ b/frontend/src/metabase/components/form/CustomForm/CustomFormMessage.tsx @@ -20,6 +20,9 @@ function CustomFormMessage({ return null; } +/** + * @deprecated + */ const CustomFormMessageLegacyContext = ( props: CustomFormMessageProps, context: CustomFormLegacyContext, diff --git a/frontend/src/metabase/components/form/CustomForm/CustomFormSection.tsx b/frontend/src/metabase/components/form/CustomForm/CustomFormSection.tsx index 833f1e340c1b..edc83572a40c 100644 --- a/frontend/src/metabase/components/form/CustomForm/CustomFormSection.tsx +++ b/frontend/src/metabase/components/form/CustomForm/CustomFormSection.tsx @@ -38,6 +38,9 @@ interface CustomFormSectionProps extends SectionProps { collapsible?: boolean; } +/** + * @deprecated + */ function CustomFormSection({ collapsible, ...props }: CustomFormSectionProps) { const Section = collapsible ? CollapsibleSection : StandardSection; return
; diff --git a/frontend/src/metabase/components/form/CustomForm/CustomFormSubmit.tsx b/frontend/src/metabase/components/form/CustomForm/CustomFormSubmit.tsx index a30f5eacf6fd..f3cbe5b5b3c5 100644 --- a/frontend/src/metabase/components/form/CustomForm/CustomFormSubmit.tsx +++ b/frontend/src/metabase/components/form/CustomForm/CustomFormSubmit.tsx @@ -50,6 +50,9 @@ function CustomFormSubmit({ ); } +/** + * @deprecated + */ const CustomFormSubmitLegacyContext = ( props: CustomFormSubmitProps, context: CustomFormLegacyContext, diff --git a/frontend/src/metabase/components/form/CustomForm/Form.tsx b/frontend/src/metabase/components/form/CustomForm/Form.tsx index b63362ad90ac..4d4233a3e411 100644 --- a/frontend/src/metabase/components/form/CustomForm/Form.tsx +++ b/frontend/src/metabase/components/form/CustomForm/Form.tsx @@ -19,6 +19,9 @@ function Form({ ); } +/** + * @deprecated + */ const FormUsingLegacyContext = ( props: Props, context: CustomFormLegacyContext, diff --git a/frontend/src/metabase/components/form/FormField/FormField.tsx b/frontend/src/metabase/components/form/FormField/FormField.tsx index d46fbcfed925..1c2aed106c8f 100644 --- a/frontend/src/metabase/components/form/FormField/FormField.tsx +++ b/frontend/src/metabase/components/form/FormField/FormField.tsx @@ -58,6 +58,9 @@ function getHorizontalPropValue( return false; } +/** + * @deprecated + */ function FormField({ className, formField, diff --git a/frontend/src/metabase/components/form/FormMessage/FormMessage.tsx b/frontend/src/metabase/components/form/FormMessage/FormMessage.tsx index 8afe9e27123f..f16635056179 100644 --- a/frontend/src/metabase/components/form/FormMessage/FormMessage.tsx +++ b/frontend/src/metabase/components/form/FormMessage/FormMessage.tsx @@ -32,6 +32,9 @@ const getMessage = ({ return getSuccessMessage(formSuccess); }; +/** + * @deprecated + */ export const getErrorMessage = (formError?: Response) => { if (formError) { if (formError.data && formError.data.message) { @@ -44,10 +47,16 @@ export const getErrorMessage = (formError?: Response) => { } }; +/** + * @deprecated + */ export const getSuccessMessage = (formSuccess?: Response) => { return formSuccess?.data?.message; }; +/** + * @deprecated + */ function FormMessage({ className, message, diff --git a/frontend/src/metabase/components/form/FormWidget.jsx b/frontend/src/metabase/components/form/FormWidget.jsx index 17da1cc9f5dc..ff907d82c232 100644 --- a/frontend/src/metabase/components/form/FormWidget.jsx +++ b/frontend/src/metabase/components/form/FormWidget.jsx @@ -5,7 +5,6 @@ import { PLUGIN_FORM_WIDGETS } from "metabase/plugins"; import FormInfoWidget from "./widgets/FormInfoWidget"; import FormInputWidget from "./widgets/FormInputWidget"; -import FormDateWidget from "./widgets/FormDateWidget"; import FormEmailWidget from "./widgets/FormEmailWidget"; import FormTextAreaWidget from "./widgets/FormTextAreaWidget"; import FormPasswordWidget from "./widgets/FormPasswordWidget"; @@ -25,7 +24,6 @@ import FormModelWidget from "./widgets/FormModelWidget"; const WIDGETS = { info: FormInfoWidget, input: FormInputWidget, - date: FormDateWidget, email: FormEmailWidget, text: FormTextAreaWidget, checkbox: FormCheckBoxWidget, @@ -52,6 +50,9 @@ export function getWidgetComponent(formField) { return formField.type || FormInputWidget; } +/** + * @deprecated + */ const FormWidget = forwardRef(function FormWidget( { field, formField, ...props }, ref, diff --git a/frontend/src/metabase/components/form/FormikCustomForm/CustomForm.tsx b/frontend/src/metabase/components/form/FormikCustomForm/CustomForm.tsx index f444eb492edc..e67f81689390 100644 --- a/frontend/src/metabase/components/form/FormikCustomForm/CustomForm.tsx +++ b/frontend/src/metabase/components/form/FormikCustomForm/CustomForm.tsx @@ -40,6 +40,9 @@ export interface CustomFormProps | ((props: FormRenderProps) => JSX.Element); } +/** + * @deprecated + */ function CustomForm( props: CustomFormProps, ) { diff --git a/frontend/src/metabase/components/form/FormikCustomForm/CustomFormField.tsx b/frontend/src/metabase/components/form/FormikCustomForm/CustomFormField.tsx index 01d233bf48b7..548b063173ba 100644 --- a/frontend/src/metabase/components/form/FormikCustomForm/CustomFormField.tsx +++ b/frontend/src/metabase/components/form/FormikCustomForm/CustomFormField.tsx @@ -45,6 +45,9 @@ function getFieldDefinition( ); } +/** + * @deprecated + */ function RawCustomFormField( props: CustomFormFieldProps & { forwardedRef?: any }, ) { @@ -112,8 +115,14 @@ function RawCustomFormField( ); } -export default React.forwardRef( - function CustomFormField(props, ref) { - return ; - }, -); +/** + * @deprecated + */ +const CustomFormField = React.forwardRef< + HTMLInputElement, + CustomFormFieldProps +>(function CustomFormField(props, ref) { + return ; +}); + +export default CustomFormField; diff --git a/frontend/src/metabase/components/form/FormikCustomForm/CustomFormFooter.tsx b/frontend/src/metabase/components/form/FormikCustomForm/CustomFormFooter.tsx index 024670d7280c..f960d09e3a5f 100644 --- a/frontend/src/metabase/components/form/FormikCustomForm/CustomFormFooter.tsx +++ b/frontend/src/metabase/components/form/FormikCustomForm/CustomFormFooter.tsx @@ -50,6 +50,9 @@ interface LegacyContextProps { } // Modal components uses legacy React context to pass `isModal` prop +/** + * @deprecated + */ const CustomFormFooterLegacyContext = ( props: CustomFormFooterProps, { isModal }: LegacyContextProps, diff --git a/frontend/src/metabase/components/form/FormikCustomForm/CustomFormMessage.tsx b/frontend/src/metabase/components/form/FormikCustomForm/CustomFormMessage.tsx index d87061ac7520..e82d5fa7acf1 100644 --- a/frontend/src/metabase/components/form/FormikCustomForm/CustomFormMessage.tsx +++ b/frontend/src/metabase/components/form/FormikCustomForm/CustomFormMessage.tsx @@ -10,6 +10,9 @@ export interface CustomFormMessageProps { noPadding?: boolean; } +/** + * @deprecated + */ function CustomFormMessage(props: CustomFormMessageProps) { const { error } = useForm(); if (error) { diff --git a/frontend/src/metabase/components/form/FormikCustomForm/CustomFormSection.tsx b/frontend/src/metabase/components/form/FormikCustomForm/CustomFormSection.tsx index 833f1e340c1b..edc83572a40c 100644 --- a/frontend/src/metabase/components/form/FormikCustomForm/CustomFormSection.tsx +++ b/frontend/src/metabase/components/form/FormikCustomForm/CustomFormSection.tsx @@ -38,6 +38,9 @@ interface CustomFormSectionProps extends SectionProps { collapsible?: boolean; } +/** + * @deprecated + */ function CustomFormSection({ collapsible, ...props }: CustomFormSectionProps) { const Section = collapsible ? CollapsibleSection : StandardSection; return
; diff --git a/frontend/src/metabase/components/form/FormikCustomForm/CustomFormSubmit.tsx b/frontend/src/metabase/components/form/FormikCustomForm/CustomFormSubmit.tsx index 9bf5879c66c6..f3d4c8e3c4fd 100644 --- a/frontend/src/metabase/components/form/FormikCustomForm/CustomFormSubmit.tsx +++ b/frontend/src/metabase/components/form/FormikCustomForm/CustomFormSubmit.tsx @@ -13,6 +13,9 @@ export interface CustomFormSubmitProps { fullWidth?: boolean; } +/** + * @deprecated + */ function CustomFormSubmit(props: CustomFormSubmitProps) { const { submitting, diff --git a/frontend/src/metabase/components/form/FormikFormField/FormField.tsx b/frontend/src/metabase/components/form/FormikFormField/FormField.tsx index 6ac423034077..b0af24d92728 100644 --- a/frontend/src/metabase/components/form/FormikFormField/FormField.tsx +++ b/frontend/src/metabase/components/form/FormikFormField/FormField.tsx @@ -60,6 +60,9 @@ function getHorizontalPropValue( return false; } +/** + * @deprecated + */ function FormField({ className, formField, diff --git a/frontend/src/metabase/components/form/FormikStandardForm.tsx b/frontend/src/metabase/components/form/FormikStandardForm.tsx index 349b288cfe75..3c9405ad89ff 100644 --- a/frontend/src/metabase/components/form/FormikStandardForm.tsx +++ b/frontend/src/metabase/components/form/FormikStandardForm.tsx @@ -14,6 +14,9 @@ interface Props onClose?: () => void; } +/** + * @deprecated + */ function StandardForm({ submitTitle, submitFullWidth, diff --git a/frontend/src/metabase/components/form/StandardForm.tsx b/frontend/src/metabase/components/form/StandardForm.tsx index 0bcb94f26e5a..2e9df996f600 100644 --- a/frontend/src/metabase/components/form/StandardForm.tsx +++ b/frontend/src/metabase/components/form/StandardForm.tsx @@ -10,6 +10,9 @@ interface Props extends BaseFormProps, CustomFormFooterProps { onClose?: () => void; } +/** + * @deprecated + */ const StandardForm = ({ submitTitle, submitFullWidth, diff --git a/frontend/src/metabase/components/form/widgets/FormDateWidget/FormDateWidget.tsx b/frontend/src/metabase/components/form/widgets/FormDateWidget/FormDateWidget.tsx deleted file mode 100644 index 71c26c712e02..000000000000 --- a/frontend/src/metabase/components/form/widgets/FormDateWidget/FormDateWidget.tsx +++ /dev/null @@ -1,85 +0,0 @@ -import React, { forwardRef, Ref, useCallback, useMemo } from "react"; -import { Moment } from "moment-timezone"; -import { - getNumericDateStyleFromSettings, - getTimeStyleFromSettings, - has24HourModeSetting, - parseTimestamp, -} from "metabase/lib/time"; -import DateWidget from "metabase/core/components/DateWidget"; -import { FormField } from "./types"; - -export interface FormDateWidgetProps { - field: FormField; - placeholder?: string; - values: Record; - readOnly?: boolean; - autoFocus?: boolean; - tabIndex?: number; - hasTimeField?: string; - onChangeField?: (field: string, value: unknown) => void; -} - -const FormDateWidget = forwardRef(function FormDateWidget( - { - field, - placeholder, - values, - readOnly, - autoFocus, - tabIndex, - hasTimeField = "", - onChangeField, - }: FormDateWidgetProps, - ref: Ref, -) { - const value = useMemo(() => { - return field.value ? parseTimestamp(field.value) : undefined; - }, [field]); - - const handleFocus = useCallback(() => { - field.onFocus?.(field.value); - }, [field]); - - const handleBlur = useCallback(() => { - field.onBlur?.(field.value); - }, [field]); - - const handleChange = useCallback( - (newValue?: Moment) => { - field.onChange?.(newValue?.format()); - }, - [field], - ); - - const handleHasTimeChange = useCallback( - (hasTime: boolean) => { - onChangeField?.(hasTimeField, hasTime); - }, - [hasTimeField, onChangeField], - ); - - return ( - - ); -}); - -export default FormDateWidget; diff --git a/frontend/src/metabase/components/form/widgets/FormDateWidget/index.ts b/frontend/src/metabase/components/form/widgets/FormDateWidget/index.ts deleted file mode 100644 index df005321703a..000000000000 --- a/frontend/src/metabase/components/form/widgets/FormDateWidget/index.ts +++ /dev/null @@ -1 +0,0 @@ -export { default } from "./FormDateWidget"; diff --git a/frontend/src/metabase/components/form/widgets/FormDateWidget/types.ts b/frontend/src/metabase/components/form/widgets/FormDateWidget/types.ts deleted file mode 100644 index 26db3ee8eae3..000000000000 --- a/frontend/src/metabase/components/form/widgets/FormDateWidget/types.ts +++ /dev/null @@ -1,10 +0,0 @@ -export interface FormField { - name: string; - value?: string; - visited?: boolean; - active?: boolean; - error?: string; - onChange?: (value?: string) => void; - onFocus?: (value?: string) => void; - onBlur?: (value?: string) => void; -} diff --git a/frontend/src/metabase/containers/Form.jsx b/frontend/src/metabase/containers/Form.jsx index bf136dad065f..53ad7c61952b 100644 --- a/frontend/src/metabase/containers/Form.jsx +++ b/frontend/src/metabase/containers/Form.jsx @@ -56,6 +56,9 @@ const ReduxFormComponent = reduxForm()( }, ); +/** + * @deprecated + */ class Form extends React.Component { _state = { submitting: false, @@ -288,4 +291,9 @@ class Form extends React.Component { } } -export default connect(makeMapStateToProps)(Form); +/** + * @deprecated + */ +const DeprecatedForm = connect(makeMapStateToProps)(Form); + +export default DeprecatedForm; diff --git a/frontend/src/metabase/containers/FormikForm/FormikForm.tsx b/frontend/src/metabase/containers/FormikForm/FormikForm.tsx index 6a405cd10d70..817b9904f530 100644 --- a/frontend/src/metabase/containers/FormikForm/FormikForm.tsx +++ b/frontend/src/metabase/containers/FormikForm/FormikForm.tsx @@ -33,6 +33,7 @@ interface FormContainerProps initial?: () => void; normalize?: () => void; + onValuesChange?: (newValues: Record) => void; onSubmit: ( values: Values, formikHelpers?: FormikHelpers, @@ -83,6 +84,9 @@ function getGeneralErrorMessage(error: ServerErrorResponse) { } } +/** + * @deprecated + */ function Form({ form, fields, @@ -92,6 +96,7 @@ function Form({ validate, initial, normalize, + onValuesChange, onSubmit, onSubmitSuccess, ...props @@ -99,6 +104,11 @@ function Form({ const [error, setError] = useState(null); const [values, setValues] = useState({}); + const handleValuesChange = (newValues: any) => { + onValuesChange?.(newValues); + setValues(newValues); + }; + const { inlineFields, registerFormField, unregisterFormField } = useInlineFields(); @@ -249,7 +259,7 @@ function Form({ error={error} registerFormField={registerFormField} unregisterFormField={unregisterFormField} - onValuesChange={setValues} + onValuesChange={handleValuesChange} /> )} diff --git a/frontend/src/metabase/containers/SaveQuestionModal.jsx b/frontend/src/metabase/containers/SaveQuestionModal.jsx index 88cfed4f8738..6ae7cdffa799 100644 --- a/frontend/src/metabase/containers/SaveQuestionModal.jsx +++ b/frontend/src/metabase/containers/SaveQuestionModal.jsx @@ -14,6 +14,16 @@ import { generateQueryDescription } from "metabase-lib/queries/utils/description import "./SaveQuestionModal.css"; +const getSingleStepTitle = (questionType, showSaveType) => { + if (questionType === "model") { + return t`Save model`; + } else if (showSaveType) { + return t`Save question`; + } else { + return t`Save new question`; + } +}; + export default class SaveQuestionModal extends Component { static propTypes = { card: PropTypes.object.isRequired, @@ -104,17 +114,16 @@ export default class SaveQuestionModal extends Component { ? t`First, save your question` : t`First, save your model`; - const singleStepTitle = - questionType === "question" ? t`Save question` : t`Save model`; - - const title = this.props.multiStep ? multiStepTitle : singleStepTitle; - const showSaveType = !card.id && !!originalCard && !originalCard.dataset && originalCard.can_write; + const singleStepTitle = getSingleStepTitle(questionType, showSaveType); + + const title = this.props.multiStep ? multiStepTitle : singleStepTitle; + const nameInputPlaceholder = questionType === "question" ? t`What is the name of your question?` diff --git a/frontend/src/metabase/core/components/AccordionList/AccordionListCell.jsx b/frontend/src/metabase/core/components/AccordionList/AccordionListCell.jsx index 52f66c7c76ae..74c914068f0c 100644 --- a/frontend/src/metabase/core/components/AccordionList/AccordionListCell.jsx +++ b/frontend/src/metabase/core/components/AccordionList/AccordionListCell.jsx @@ -117,6 +117,7 @@ export const AccordionListCell = ({ content = ( ) => void; } +interface CheckboxTooltipProps { + hasTooltip: boolean; + tooltipLabel: ReactNode; + children: ReactNode; +} + const CheckBox = forwardRef(function Checkbox( { name, + id, label, labelEllipsis = false, checked, @@ -75,7 +82,7 @@ const CheckBox = forwardRef(function Checkbox( tooltipLabel={label} > (function Checkbox( ); }); -interface CheckboxTooltipProps { - hasTooltip: boolean; - tooltipLabel: ReactNode; - children: ReactNode; -} - function CheckboxTooltip({ hasTooltip, tooltipLabel, diff --git a/frontend/src/metabase/core/components/ColorInput/ColorInput.stories.tsx b/frontend/src/metabase/core/components/ColorInput/ColorInput.stories.tsx index f972aae67e67..71ab0b1e9bf4 100644 --- a/frontend/src/metabase/core/components/ColorInput/ColorInput.stories.tsx +++ b/frontend/src/metabase/core/components/ColorInput/ColorInput.stories.tsx @@ -1,5 +1,5 @@ import React, { useState } from "react"; -import { ComponentStory } from "@storybook/react"; +import type { ComponentStory } from "@storybook/react"; import { useArgs } from "@storybook/client-api"; import ColorInput from "./ColorInput"; diff --git a/frontend/src/metabase/core/components/ColorPicker/ColorPicker.stories.tsx b/frontend/src/metabase/core/components/ColorPicker/ColorPicker.stories.tsx index 93e7f50e7781..f9f613e0801b 100644 --- a/frontend/src/metabase/core/components/ColorPicker/ColorPicker.stories.tsx +++ b/frontend/src/metabase/core/components/ColorPicker/ColorPicker.stories.tsx @@ -1,5 +1,5 @@ import React from "react"; -import { ComponentStory } from "@storybook/react"; +import type { ComponentStory } from "@storybook/react"; import { useArgs } from "@storybook/client-api"; import { color } from "metabase/lib/colors"; import ColorPicker from "./ColorPicker"; diff --git a/frontend/src/metabase/core/components/ColorPill/ColorPill.stories.tsx b/frontend/src/metabase/core/components/ColorPill/ColorPill.stories.tsx index 7033efe4d50d..e1a962c620b7 100644 --- a/frontend/src/metabase/core/components/ColorPill/ColorPill.stories.tsx +++ b/frontend/src/metabase/core/components/ColorPill/ColorPill.stories.tsx @@ -1,5 +1,5 @@ import React from "react"; -import { ComponentStory } from "@storybook/react"; +import type { ComponentStory } from "@storybook/react"; import { color } from "metabase/lib/colors"; import ColorPill from "./ColorPill"; diff --git a/frontend/src/metabase/core/components/ColorPill/ColorPill.styled.tsx b/frontend/src/metabase/core/components/ColorPill/ColorPill.styled.tsx index be38503853be..2d290c88277f 100644 --- a/frontend/src/metabase/core/components/ColorPill/ColorPill.styled.tsx +++ b/frontend/src/metabase/core/components/ColorPill/ColorPill.styled.tsx @@ -1,9 +1,14 @@ import styled from "@emotion/styled"; +import { css } from "@emotion/react"; + import { color } from "metabase/lib/colors"; +import { PillSize } from "./types"; + export interface ColorPillRootProps { isAuto: boolean; isSelected: boolean; + pillSize: PillSize; } export const ColorPillRoot = styled.div` @@ -21,6 +26,17 @@ export const ColorPillRoot = styled.div` border-color: ${props => props.isSelected ? color("text-dark") : color("text-light")}; } + + ${props => + props.pillSize === "small" && + css` + padding: 1px; + + ${ColorPillContent} { + height: 0.875rem; + width: 0.875rem; + } + `}; `; export const ColorPillContent = styled.div` diff --git a/frontend/src/metabase/core/components/ColorPill/ColorPill.tsx b/frontend/src/metabase/core/components/ColorPill/ColorPill.tsx index 071ca11ec534..3b472b1c5076 100644 --- a/frontend/src/metabase/core/components/ColorPill/ColorPill.tsx +++ b/frontend/src/metabase/core/components/ColorPill/ColorPill.tsx @@ -6,6 +6,7 @@ import React, { useCallback, } from "react"; import { ColorPillContent, ColorPillRoot } from "./ColorPill.styled"; +import { PillSize } from "./types"; export type ColorPillAttributes = Omit< HTMLAttributes, @@ -17,6 +18,7 @@ export interface ColorPillProps extends ColorPillAttributes { isAuto?: boolean; isSelected?: boolean; onSelect?: (newColor: string) => void; + pillSize?: PillSize; } const ColorPill = forwardRef(function ColorPill( @@ -25,6 +27,7 @@ const ColorPill = forwardRef(function ColorPill( isAuto = false, isSelected = true, "aria-label": ariaLabel = color, + pillSize = "medium", onClick, onSelect, ...props @@ -47,10 +50,14 @@ const ColorPill = forwardRef(function ColorPill( isSelected={isSelected} aria-label={ariaLabel} onClick={handleClick} + pillSize={pillSize} > ); }); -export default ColorPill; +export default Object.assign(ColorPill, { + Content: ColorPillContent, + Root: ColorPillRoot, +}); diff --git a/frontend/src/metabase/core/components/ColorPill/types.ts b/frontend/src/metabase/core/components/ColorPill/types.ts new file mode 100644 index 000000000000..901639b4519b --- /dev/null +++ b/frontend/src/metabase/core/components/ColorPill/types.ts @@ -0,0 +1 @@ +export type PillSize = "small" | "medium"; diff --git a/frontend/src/metabase/core/components/ColorRange/ColorRange.stories.tsx b/frontend/src/metabase/core/components/ColorRange/ColorRange.stories.tsx index 6a3ccf982c01..78405b0640cb 100644 --- a/frontend/src/metabase/core/components/ColorRange/ColorRange.stories.tsx +++ b/frontend/src/metabase/core/components/ColorRange/ColorRange.stories.tsx @@ -1,5 +1,5 @@ import React from "react"; -import { ComponentStory } from "@storybook/react"; +import type { ComponentStory } from "@storybook/react"; import { color } from "metabase/lib/colors"; import ColorRange from "./ColorRange"; diff --git a/frontend/src/metabase/core/components/ColorRangeSelector/ColorRangeSelector.stories.tsx b/frontend/src/metabase/core/components/ColorRangeSelector/ColorRangeSelector.stories.tsx index 4cd03ddb7022..09c23d4d8b41 100644 --- a/frontend/src/metabase/core/components/ColorRangeSelector/ColorRangeSelector.stories.tsx +++ b/frontend/src/metabase/core/components/ColorRangeSelector/ColorRangeSelector.stories.tsx @@ -1,5 +1,5 @@ import React from "react"; -import { ComponentStory } from "@storybook/react"; +import type { ComponentStory } from "@storybook/react"; import { useArgs } from "@storybook/client-api"; import { color } from "metabase/lib/colors"; import ColorRangeSelector from "./ColorRangeSelector"; diff --git a/frontend/src/metabase/core/components/ColorSelector/ColorSelector.stories.tsx b/frontend/src/metabase/core/components/ColorSelector/ColorSelector.stories.tsx index c3594952a35e..e0c0983618fe 100644 --- a/frontend/src/metabase/core/components/ColorSelector/ColorSelector.stories.tsx +++ b/frontend/src/metabase/core/components/ColorSelector/ColorSelector.stories.tsx @@ -1,5 +1,5 @@ import React from "react"; -import { ComponentStory } from "@storybook/react"; +import type { ComponentStory } from "@storybook/react"; import { useArgs } from "@storybook/client-api"; import { color } from "metabase/lib/colors"; import ColorSelector from "./ColorSelector"; diff --git a/frontend/src/metabase/core/components/DateInput/DateInput.stories.tsx b/frontend/src/metabase/core/components/DateInput/DateInput.stories.tsx index a065528f4afc..98be0c7066f5 100644 --- a/frontend/src/metabase/core/components/DateInput/DateInput.stories.tsx +++ b/frontend/src/metabase/core/components/DateInput/DateInput.stories.tsx @@ -1,6 +1,6 @@ import React, { useState } from "react"; import { Moment } from "moment-timezone"; -import { ComponentStory } from "@storybook/react"; +import type { ComponentStory } from "@storybook/react"; import DateInput from "./DateInput"; export default { diff --git a/frontend/src/metabase/core/components/DateSelector/DateSelector.stories.tsx b/frontend/src/metabase/core/components/DateSelector/DateSelector.stories.tsx index 48afcb2a5732..3efa8ed158b7 100644 --- a/frontend/src/metabase/core/components/DateSelector/DateSelector.stories.tsx +++ b/frontend/src/metabase/core/components/DateSelector/DateSelector.stories.tsx @@ -1,6 +1,6 @@ import React, { useState } from "react"; import moment, { Moment } from "moment-timezone"; -import { ComponentStory } from "@storybook/react"; +import type { ComponentStory } from "@storybook/react"; import DateSelector from "./DateSelector"; export default { diff --git a/frontend/src/metabase/core/components/DateSelector/DateSelector.tsx b/frontend/src/metabase/core/components/DateSelector/DateSelector.tsx index 54d2ce6035e2..43ca94a73aeb 100644 --- a/frontend/src/metabase/core/components/DateSelector/DateSelector.tsx +++ b/frontend/src/metabase/core/components/DateSelector/DateSelector.tsx @@ -22,7 +22,7 @@ export interface DateSelectorProps { style?: CSSProperties; value?: Moment; hasTime?: boolean; - is24HourMode?: boolean; + timeFormat?: string; onChange?: (date?: Moment) => void; onHasTimeChange?: (hasTime: boolean) => void; onSubmit?: () => void; @@ -34,7 +34,7 @@ const DateSelector = forwardRef(function DateSelector( style, value, hasTime, - is24HourMode, + timeFormat, onChange, onHasTimeChange, onSubmit, @@ -79,7 +79,7 @@ const DateSelector = forwardRef(function DateSelector( diff --git a/frontend/src/metabase/core/components/DateWidget/DateWidget.stories.tsx b/frontend/src/metabase/core/components/DateWidget/DateWidget.stories.tsx index c602f64ced23..d5beae43f1c6 100644 --- a/frontend/src/metabase/core/components/DateWidget/DateWidget.stories.tsx +++ b/frontend/src/metabase/core/components/DateWidget/DateWidget.stories.tsx @@ -1,6 +1,6 @@ import React, { useState } from "react"; import { Moment } from "moment-timezone"; -import { ComponentStory } from "@storybook/react"; +import type { ComponentStory } from "@storybook/react"; import DateWidget from "./DateWidget"; export default { diff --git a/frontend/src/metabase/core/components/DateWidget/DateWidget.tsx b/frontend/src/metabase/core/components/DateWidget/DateWidget.tsx index ce9f2b4b2b98..5fd96ec5565f 100644 --- a/frontend/src/metabase/core/components/DateWidget/DateWidget.tsx +++ b/frontend/src/metabase/core/components/DateWidget/DateWidget.tsx @@ -20,7 +20,6 @@ export interface DateWidgetProps extends DateWidgetAttributes { hasTime?: boolean; dateFormat?: string; timeFormat?: string; - is24HourMode?: boolean; error?: boolean; fullWidth?: boolean; onChange?: (date?: Moment) => void; @@ -33,7 +32,6 @@ const DateWidget = forwardRef(function DateWidget( hasTime, dateFormat, timeFormat, - is24HourMode, error, fullWidth, onChange, @@ -60,13 +58,13 @@ const DateWidget = forwardRef(function DateWidget( } - onHide={handleClose} + onClickOutside={handleClose} > , "onSubmit" | "onReset"> { + disabled?: boolean; +} + +const Form = forwardRef(function Form( + { disabled, ...props }: FormProps, + ref: Ref, +) { + const { handleSubmit, handleReset } = useFormikContext(); + + return ( + + ); +}); + +const handleDisabledEvent = (event: SyntheticEvent) => { + event.preventDefault(); + event.stopPropagation(); +}; + +export default Form; diff --git a/frontend/src/metabase/core/components/Form/index.ts b/frontend/src/metabase/core/components/Form/index.ts new file mode 100644 index 000000000000..d0066d51caf9 --- /dev/null +++ b/frontend/src/metabase/core/components/Form/index.ts @@ -0,0 +1,2 @@ +export { default } from "./Form"; +export type { FormProps } from "./Form"; diff --git a/frontend/src/metabase/core/components/FormCheckBox/FormCheckBox.tsx b/frontend/src/metabase/core/components/FormCheckBox/FormCheckBox.tsx index fcfa83258c60..333e8ffd9489 100644 --- a/frontend/src/metabase/core/components/FormCheckBox/FormCheckBox.tsx +++ b/frontend/src/metabase/core/components/FormCheckBox/FormCheckBox.tsx @@ -1,27 +1,44 @@ -import React, { forwardRef, Ref } from "react"; +import React, { forwardRef, ReactNode, Ref } from "react"; import { useField } from "formik"; +import { useUniqueId } from "metabase/hooks/use-unique-id"; import CheckBox, { CheckBoxProps } from "metabase/core/components/CheckBox"; +import FormField from "metabase/core/components/FormField"; export interface FormCheckBoxProps extends Omit { name: string; + title?: string; + description?: ReactNode; } const FormCheckBox = forwardRef(function FormCheckBox( - { name, ...props }: FormCheckBoxProps, - ref: Ref, + { name, className, style, title, description, ...props }: FormCheckBoxProps, + ref: Ref, ) { - const [{ value, onChange, onBlur }] = useField(name); + const id = useUniqueId(); + const [{ value, onChange, onBlur }, { error, touched }] = useField(name); return ( - + className={className} + style={style} + title={title} + description={description} + alignment="start" + orientation="horizontal" + htmlFor={id} + error={touched ? error : undefined} + > + + ); }); diff --git a/frontend/src/metabase/core/components/FormCheckBox/FormCheckBox.unit.spec.tsx b/frontend/src/metabase/core/components/FormCheckBox/FormCheckBox.unit.spec.tsx index f596c797f078..d45b3ca23797 100644 --- a/frontend/src/metabase/core/components/FormCheckBox/FormCheckBox.unit.spec.tsx +++ b/frontend/src/metabase/core/components/FormCheckBox/FormCheckBox.unit.spec.tsx @@ -3,10 +3,9 @@ import { Form, Formik } from "formik"; import * as Yup from "yup"; import { render, screen, waitFor } from "@testing-library/react"; import userEvent from "@testing-library/user-event"; -import FormField from "metabase/core/components/FormField"; import FormCheckBox from "./FormCheckBox"; -const TEST_SCHEMA = Yup.object().shape({ +const TEST_SCHEMA = Yup.object({ value: Yup.boolean().isTrue("error"), }); @@ -26,9 +25,7 @@ const TestFormCheckBox = ({ onSubmit={onSubmit} > - - - + @@ -44,14 +41,17 @@ describe("FormCheckBox", () => { expect(screen.getByRole("checkbox")).toBeChecked(); }); - it("should propagate the changed value to the form", () => { + it("should propagate the changed value to the form", async () => { const onSubmit = jest.fn(); render(); userEvent.click(screen.getByRole("checkbox")); userEvent.click(screen.getByText("Submit")); - waitFor(() => expect(onSubmit).toHaveBeenCalledWith({ value: true })); + await waitFor(() => { + const values = { value: true }; + expect(onSubmit).toHaveBeenCalledWith(values, expect.anything()); + }); }); it("should be referenced by the label", () => { @@ -62,13 +62,13 @@ describe("FormCheckBox", () => { expect(screen.getByLabelText("Label")).toBeInTheDocument(); }); - it("should be validated on blur", () => { + it("should be validated on blur", async () => { const onSubmit = jest.fn(); render(); userEvent.click(screen.getByRole("checkbox")); userEvent.tab(); - waitFor(() => expect(screen.getByText("Label: error")).toBeInTheDocument()); + expect(await screen.findByText(": error")).toBeInTheDocument(); }); }); diff --git a/frontend/src/metabase/core/components/FormDateInput/FormDateInput.tsx b/frontend/src/metabase/core/components/FormDateInput/FormDateInput.tsx new file mode 100644 index 000000000000..1d91ca729591 --- /dev/null +++ b/frontend/src/metabase/core/components/FormDateInput/FormDateInput.tsx @@ -0,0 +1,75 @@ +import React, { forwardRef, ReactNode, Ref, useCallback, useMemo } from "react"; +import moment, { Moment } from "moment-timezone"; +import { useField } from "formik"; +import { useUniqueId } from "metabase/hooks/use-unique-id"; +import DateWidget, { + DateWidgetProps, +} from "metabase/core/components/DateWidget"; +import FormField from "metabase/core/components/FormField"; + +export interface FormDateInputProps + extends Omit< + DateWidgetProps, + "value" | "error" | "fullWidth" | "onChange" | "onBlur" + > { + name: string; + title?: string; + description?: ReactNode; + nullable?: boolean; +} + +const FormDateInput = forwardRef(function FormDateInput( + { + name, + className, + style, + title, + description, + nullable, + ...props + }: FormDateInputProps, + ref: Ref, +) { + const id = useUniqueId(); + const [{ value, onBlur }, { error, touched }, { setValue }] = useField(name); + + const date = useMemo(() => { + return value ? moment.parseZone(value) : undefined; + }, [value]); + + const handleChange = useCallback( + (date: Moment | undefined) => { + if (date) { + setValue(date.toISOString(true)); + } else { + setValue(nullable ? null : undefined); + } + }, + [nullable, setValue], + ); + + return ( + + + + ); +}); + +export default FormDateInput; diff --git a/frontend/src/metabase/core/components/FormDateInput/FormDateInput.unit.spec.tsx b/frontend/src/metabase/core/components/FormDateInput/FormDateInput.unit.spec.tsx new file mode 100644 index 000000000000..b54387402966 --- /dev/null +++ b/frontend/src/metabase/core/components/FormDateInput/FormDateInput.unit.spec.tsx @@ -0,0 +1,74 @@ +import React from "react"; +import { Form, Formik } from "formik"; +import * as Yup from "yup"; +import { render, screen, waitFor } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import FormDateInput from "./FormDateInput"; + +const TEST_SCHEMA = Yup.object({ + value: Yup.string().required("error"), +}); + +interface TestFormDateInputProps { + initialValue?: string; + onSubmit: () => void; +} + +const TestFormDateInput = ({ + initialValue, + onSubmit, +}: TestFormDateInputProps) => { + return ( + +
+ + + + + ); +}; + +describe("FormDateInput", () => { + it("should use the initial value from the form", () => { + const onSubmit = jest.fn(); + + render(); + + expect(screen.getByRole("textbox")).toHaveValue("10/20/2022"); + }); + + it("should propagate the changed value to the form", async () => { + const onSubmit = jest.fn(); + + render(); + userEvent.type(screen.getByRole("textbox"), "10/20/22"); + userEvent.click(screen.getByText("Submit")); + + await waitFor(() => { + const value = expect.stringMatching(/2022-10-20T00:00:00.000/); + expect(onSubmit).toHaveBeenCalledWith({ value }, expect.anything()); + }); + }); + + it("should be referenced by the label", () => { + const onSubmit = jest.fn(); + + render(); + + expect(screen.getByLabelText("Date")).toBeInTheDocument(); + }); + + it("should be validated on blur", async () => { + const onSubmit = jest.fn(); + + render(); + userEvent.clear(screen.getByRole("textbox")); + userEvent.tab(); + + expect(await screen.findByText(": error")).toBeInTheDocument(); + }); +}); diff --git a/frontend/src/metabase/core/components/FormDateInput/index.ts b/frontend/src/metabase/core/components/FormDateInput/index.ts new file mode 100644 index 000000000000..58ce400611d6 --- /dev/null +++ b/frontend/src/metabase/core/components/FormDateInput/index.ts @@ -0,0 +1,2 @@ +export { default } from "./FormDateInput"; +export type { FormDateInputProps } from "./FormDateInput"; diff --git a/frontend/src/metabase/core/components/FormErrorMessage/FormErrorMessage.styled.tsx b/frontend/src/metabase/core/components/FormErrorMessage/FormErrorMessage.styled.tsx index b192d4034703..ce5debaeb693 100644 --- a/frontend/src/metabase/core/components/FormErrorMessage/FormErrorMessage.styled.tsx +++ b/frontend/src/metabase/core/components/FormErrorMessage/FormErrorMessage.styled.tsx @@ -1,7 +1,11 @@ import styled from "@emotion/styled"; import { color } from "metabase/lib/colors"; -export const ErrorMessageRoot = styled.div` +export interface ErrorMessageRootProps { + inline?: boolean; +} + +export const ErrorMessageRoot = styled.div` color: ${color("error")}; - margin-top: 1em; + margin-top: ${props => !props.inline && "1rem"}; `; diff --git a/frontend/src/metabase/core/components/FormErrorMessage/FormErrorMessage.tsx b/frontend/src/metabase/core/components/FormErrorMessage/FormErrorMessage.tsx index 3d1b04c34dd5..d3bffcfa0898 100644 --- a/frontend/src/metabase/core/components/FormErrorMessage/FormErrorMessage.tsx +++ b/frontend/src/metabase/core/components/FormErrorMessage/FormErrorMessage.tsx @@ -2,13 +2,13 @@ import React, { forwardRef, HTMLAttributes, Ref } from "react"; import useFormErrorMessage from "metabase/core/hooks/use-form-error-message"; import { ErrorMessageRoot } from "./FormErrorMessage.styled"; -export type FormErrorMessageProps = Omit< - HTMLAttributes, - "children" ->; +export interface FormErrorMessageProps + extends Omit, "children"> { + inline?: boolean; +} const FormErrorMessage = forwardRef(function FormErrorMessage( - props: FormErrorMessageProps, + { inline, ...props }: FormErrorMessageProps, ref: Ref, ) { const message = useFormErrorMessage(); @@ -17,7 +17,7 @@ const FormErrorMessage = forwardRef(function FormErrorMessage( } return ( - + {message} ); diff --git a/frontend/src/metabase/core/components/InputField/InputField.styled.tsx b/frontend/src/metabase/core/components/FormField/FormField.styled.tsx similarity index 73% rename from frontend/src/metabase/core/components/InputField/InputField.styled.tsx rename to frontend/src/metabase/core/components/FormField/FormField.styled.tsx index be942547e399..59427ab223a7 100644 --- a/frontend/src/metabase/core/components/InputField/InputField.styled.tsx +++ b/frontend/src/metabase/core/components/FormField/FormField.styled.tsx @@ -1,5 +1,6 @@ import styled from "@emotion/styled"; import { color } from "metabase/lib/colors"; +import Icon from "metabase/components/Icon"; import { FieldAlignment, FieldOrientation } from "./types"; export const FieldLabelError = styled.span` @@ -31,8 +32,6 @@ export interface FormCaptionProps { } export const FieldCaption = styled.div` - display: flex; - align-items: center; margin-left: ${props => props.orientation === "horizontal" && props.alignment === "start" && @@ -41,7 +40,6 @@ export const FieldCaption = styled.div` props.orientation === "horizontal" && props.alignment === "end" && "0.5rem"}; - margin-bottom: 0.5rem; `; export const FieldLabel = styled.label` @@ -50,6 +48,30 @@ export const FieldLabel = styled.label` font-weight: 900; `; +export const FieldLabelContainer = styled.div` + display: flex; + align-items: center; + margin-bottom: 0.5em; +`; + export const FieldDescription = styled.div` margin-bottom: 0.5rem; `; + +export const FieldInfoIcon = styled(Icon)` + color: ${color("bg-dark")}; + margin-left: 0.5rem; + width: 0.75rem; + height: 0.75rem; + + &:hover { + color: ${() => color("brand")}; + } +`; + +export const FieldInfoLabel = styled.div` + color: ${color("text-medium")}; + font-size: 0.75rem; + margin-left: auto; + cursor: default; +`; diff --git a/frontend/src/metabase/core/components/FormField/FormField.tsx b/frontend/src/metabase/core/components/FormField/FormField.tsx index 16288cfdc9f1..9e45d57a9138 100644 --- a/frontend/src/metabase/core/components/FormField/FormField.tsx +++ b/frontend/src/metabase/core/components/FormField/FormField.tsx @@ -1,28 +1,77 @@ -import React, { forwardRef, Ref } from "react"; -import { useField } from "formik"; -import InputField, { - InputFieldProps, -} from "metabase/core/components/InputField"; +import React, { forwardRef, HTMLAttributes, ReactNode, Ref } from "react"; +import Tooltip from "metabase/components/Tooltip"; +import { FieldAlignment, FieldOrientation } from "./types"; +import { + FieldCaption, + FieldDescription, + FieldInfoIcon, + FieldInfoLabel, + FieldLabel, + FieldLabelContainer, + FieldLabelError, + FieldRoot, +} from "./FormField.styled"; -export interface FormFieldProps - extends Omit { - name: string; +export interface FormFieldProps extends HTMLAttributes { + title?: string; + description?: ReactNode; + alignment?: FieldAlignment; + orientation?: FieldOrientation; + error?: string; + htmlFor?: string; + infoLabel?: string; + infoTooltip?: string; } const FormField = forwardRef(function FormField( - { name, ...props }: FormFieldProps, + { + title, + description, + alignment = "end", + orientation = "vertical", + error, + htmlFor, + infoLabel, + infoTooltip, + children, + ...props + }: FormFieldProps, ref: Ref, ) { - const [, meta] = useField(name); - const { error, touched } = meta; + const hasError = Boolean(error); return ( - + orientation={orientation} + hasError={hasError} + > + {alignment === "start" && children} + {(title || description) && ( + + + {title && ( + + {title} + {hasError && : {error}} + + )} + {(infoLabel || infoTooltip) && ( + + {infoLabel ? ( + {infoLabel} + ) : ( + + )} + + )} + + {description && {description}} + + )} + {alignment === "end" && children} + ); }); diff --git a/frontend/src/metabase/core/components/FormField/index.ts b/frontend/src/metabase/core/components/FormField/index.ts index f1d921fd9745..68c3d0de2c0e 100644 --- a/frontend/src/metabase/core/components/FormField/index.ts +++ b/frontend/src/metabase/core/components/FormField/index.ts @@ -1,2 +1,3 @@ export { default } from "./FormField"; export type { FormFieldProps } from "./FormField"; +export type { FieldAlignment, FieldOrientation } from "./types"; diff --git a/frontend/src/metabase/core/components/FormField/types.ts b/frontend/src/metabase/core/components/FormField/types.ts index ce136dd44bb8..c2dc2f099c1f 100644 --- a/frontend/src/metabase/core/components/FormField/types.ts +++ b/frontend/src/metabase/core/components/FormField/types.ts @@ -1,2 +1,3 @@ -export type FormFieldAlignment = "start" | "end"; -export type FormFieldOrientation = "horizontal" | "vertical"; +export type FieldAlignment = "start" | "end"; + +export type FieldOrientation = "horizontal" | "vertical"; diff --git a/frontend/src/metabase/core/components/FormInput/FormInput.tsx b/frontend/src/metabase/core/components/FormInput/FormInput.tsx index e09deb094296..9a4d57e6ee01 100644 --- a/frontend/src/metabase/core/components/FormInput/FormInput.tsx +++ b/frontend/src/metabase/core/components/FormInput/FormInput.tsx @@ -1,29 +1,69 @@ -import React, { forwardRef, Ref } from "react"; +import React, { + ChangeEvent, + forwardRef, + ReactNode, + Ref, + useCallback, +} from "react"; import { useField } from "formik"; +import { useUniqueId } from "metabase/hooks/use-unique-id"; import Input, { InputProps } from "metabase/core/components/Input"; +import FormField from "metabase/core/components/FormField"; export interface FormInputProps - extends Omit { + extends Omit< + InputProps, + "value" | "error" | "fullWidth" | "onChange" | "onBlur" + > { name: string; + title?: string; + description?: ReactNode; + nullable?: boolean; } const FormInput = forwardRef(function FormInput( - { name, ...props }: FormInputProps, - ref: Ref, + { + name, + className, + style, + title, + description, + nullable, + ...props + }: FormInputProps, + ref: Ref, ) { - const [{ value, onChange, onBlur }, { error, touched }] = useField(name); + const id = useUniqueId(); + const [{ value, onBlur }, { error, touched }, { setValue }] = useField(name); + + const handleChange = useCallback( + ({ target: { value } }: ChangeEvent) => { + setValue(value === "" && nullable ? null : value); + }, + [nullable, setValue], + ); return ( - + className={className} + style={style} + title={title} + description={description} + htmlFor={id} + error={touched ? error : undefined} + > + + ); }); diff --git a/frontend/src/metabase/core/components/FormInput/FormInput.unit.spec.tsx b/frontend/src/metabase/core/components/FormInput/FormInput.unit.spec.tsx index 53eccacc52c0..7cf806be1018 100644 --- a/frontend/src/metabase/core/components/FormInput/FormInput.unit.spec.tsx +++ b/frontend/src/metabase/core/components/FormInput/FormInput.unit.spec.tsx @@ -3,10 +3,9 @@ import { Form, Formik } from "formik"; import * as Yup from "yup"; import { render, screen, waitFor } from "@testing-library/react"; import userEvent from "@testing-library/user-event"; -import FormField from "metabase/core/components/FormField"; import FormInput from "./FormInput"; -const TEST_SCHEMA = Yup.object().shape({ +const TEST_SCHEMA = Yup.object({ value: Yup.string().required("error"), }); @@ -23,9 +22,7 @@ const TestFormInput = ({ initialValue = "", onSubmit }: TestFormInputProps) => { onSubmit={onSubmit} >
- - - + @@ -41,14 +38,17 @@ describe("FormInput", () => { expect(screen.getByRole("textbox")).toHaveValue("Text"); }); - it("should propagate the changed value to the form", () => { + it("should propagate the changed value to the form", async () => { const onSubmit = jest.fn(); render(); userEvent.type(screen.getByRole("textbox"), "Text"); userEvent.click(screen.getByText("Submit")); - waitFor(() => expect(onSubmit).toHaveBeenCalledWith({ value: "Text" })); + await waitFor(() => { + const values = { value: "Text" }; + expect(onSubmit).toHaveBeenCalledWith(values, expect.anything()); + }); }); it("should be referenced by the label", () => { @@ -59,13 +59,13 @@ describe("FormInput", () => { expect(screen.getByLabelText("Label")).toBeInTheDocument(); }); - it("should be validated on blur", () => { + it("should be validated on blur", async () => { const onSubmit = jest.fn(); render(); userEvent.clear(screen.getByRole("textbox")); userEvent.tab(); - waitFor(() => expect(screen.getByText("Label: error")).toBeInTheDocument()); + expect(await screen.findByText(": error")).toBeInTheDocument(); }); }); diff --git a/frontend/src/metabase/core/components/FormNumericInput/FormNumericInput.tsx b/frontend/src/metabase/core/components/FormNumericInput/FormNumericInput.tsx index 0649432fa7e3..cad0de1e5be6 100644 --- a/frontend/src/metabase/core/components/FormNumericInput/FormNumericInput.tsx +++ b/frontend/src/metabase/core/components/FormNumericInput/FormNumericInput.tsx @@ -1,31 +1,71 @@ -import React, { forwardRef, Ref } from "react"; +import React, { + ChangeEvent, + forwardRef, + ReactNode, + Ref, + useCallback, +} from "react"; import { useField } from "formik"; +import { useUniqueId } from "metabase/hooks/use-unique-id"; import NumericInput, { NumericInputProps, } from "metabase/core/components/NumericInput"; +import FormField from "metabase/core/components/FormField"; export interface FormNumericInputProps - extends Omit { + extends Omit< + NumericInputProps, + "value" | "error" | "fullWidth" | "onChange" | "onBlur" + > { name: string; + title?: string; + description?: ReactNode; + nullable?: boolean; } const FormNumericInput = forwardRef(function FormNumericInput( - { name, ...props }: FormNumericInputProps, - ref: Ref, + { + name, + className, + style, + title, + description, + nullable, + ...props + }: FormNumericInputProps, + ref: Ref, ) { + const id = useUniqueId(); const [{ value, onBlur }, { error, touched }, { setValue }] = useField(name); + const handleChange = useCallback( + (value: number | undefined) => { + setValue(value === undefined && nullable ? null : value); + }, + [nullable, setValue], + ); + return ( - + className={className} + style={style} + title={title} + description={description} + htmlFor={id} + error={touched ? error : undefined} + > + + ); }); diff --git a/frontend/src/metabase/core/components/FormNumericInput/FormNumericInput.unit.spec.tsx b/frontend/src/metabase/core/components/FormNumericInput/FormNumericInput.unit.spec.tsx index 15ebfe2702af..04857b3ca259 100644 --- a/frontend/src/metabase/core/components/FormNumericInput/FormNumericInput.unit.spec.tsx +++ b/frontend/src/metabase/core/components/FormNumericInput/FormNumericInput.unit.spec.tsx @@ -3,10 +3,9 @@ import { Form, Formik } from "formik"; import * as Yup from "yup"; import { render, screen, waitFor } from "@testing-library/react"; import userEvent from "@testing-library/user-event"; -import FormField from "metabase/core/components/FormField"; import FormNumericInput from "./FormNumericInput"; -const TestSchema = Yup.object().shape({ +const TEST_SCHEMA = Yup.object({ value: Yup.number().required("error"), }); @@ -22,13 +21,11 @@ const TestFormNumericInput = ({ return (
- - - + @@ -44,14 +41,17 @@ describe("FormNumericInput", () => { expect(screen.getByRole("textbox")).toHaveValue("10"); }); - it("should propagate the changed value to the form", () => { + it("should propagate the changed value to the form", async () => { const onSubmit = jest.fn(); render(); userEvent.type(screen.getByRole("textbox"), "10"); userEvent.click(screen.getByText("Submit")); - waitFor(() => expect(onSubmit).toHaveBeenCalledWith({ value: 10 })); + await waitFor(() => { + const values = { value: 10 }; + expect(onSubmit).toHaveBeenCalledWith(values, expect.anything()); + }); }); it("should be referenced by the label", () => { @@ -62,13 +62,13 @@ describe("FormNumericInput", () => { expect(screen.getByLabelText("Label")).toBeInTheDocument(); }); - it("should be validated on blur", () => { + it("should be validated on blur", async () => { const onSubmit = jest.fn(); render(); userEvent.clear(screen.getByRole("textbox")); userEvent.tab(); - waitFor(() => expect(screen.getByText("Label: error")).toBeInTheDocument()); + expect(await screen.findByText(": error")).toBeInTheDocument(); }); }); diff --git a/frontend/src/metabase/core/components/FormProvider/FormProvider.tsx b/frontend/src/metabase/core/components/FormProvider/FormProvider.tsx new file mode 100644 index 000000000000..f31fdf266f02 --- /dev/null +++ b/frontend/src/metabase/core/components/FormProvider/FormProvider.tsx @@ -0,0 +1,41 @@ +import React from "react"; +import { Formik } from "formik"; +import type { FormikConfig } from "formik"; +import type { AnySchema } from "yup"; +import useFormSubmit from "metabase/core/hooks/use-form-submit"; +import useFormValidation from "metabase/core/hooks/use-form-validation"; +import FormContext from "metabase/core/context/FormContext"; + +export interface FormProviderProps extends FormikConfig { + validationSchema?: AnySchema; + validationContext?: C; +} + +function FormProvider({ + initialValues, + validationSchema, + validationContext, + onSubmit, + ...props +}: FormProviderProps): JSX.Element { + const { state, handleSubmit } = useFormSubmit({ onSubmit }); + const { initialErrors, handleValidate } = useFormValidation({ + initialValues, + validationSchema, + validationContext, + }); + + return ( + + + + ); +} + +export default FormProvider; diff --git a/frontend/src/metabase/core/components/FormProvider/index.ts b/frontend/src/metabase/core/components/FormProvider/index.ts new file mode 100644 index 000000000000..8f9d44981a9d --- /dev/null +++ b/frontend/src/metabase/core/components/FormProvider/index.ts @@ -0,0 +1 @@ +export { default } from "./FormProvider"; diff --git a/frontend/src/metabase/core/components/FormRadio/FormRadio.tsx b/frontend/src/metabase/core/components/FormRadio/FormRadio.tsx index 81ccfa138c60..6d650d705828 100644 --- a/frontend/src/metabase/core/components/FormRadio/FormRadio.tsx +++ b/frontend/src/metabase/core/components/FormRadio/FormRadio.tsx @@ -1,32 +1,53 @@ -import React, { forwardRef, Key, Ref } from "react"; +import React, { forwardRef, Key, ReactNode, Ref } from "react"; import { useField } from "formik"; import Radio, { RadioOption, RadioProps } from "metabase/core/components/Radio"; +import FormField from "metabase/core/components/FormField"; export interface FormRadioProps< TValue extends Key, TOption = RadioOption, -> extends Omit, "value" | "onChange" | "onBlur"> { +> extends Omit< + RadioProps, + "value" | "error" | "onChange" | "onBlur" + > { name: string; + title?: string; + description?: ReactNode; } const FormRadio = forwardRef(function FormRadio< TValue extends Key, TOption = RadioOption, >( - { name, ...props }: FormRadioProps, + { + name, + className, + style, + title, + description, + ...props + }: FormRadioProps, ref: Ref, ) { - const [{ value, onBlur }, , { setValue }] = useField(name); + const [{ value, onBlur }, { error, touched }, { setValue }] = useField(name); return ( - + className={className} + style={style} + title={title} + description={description} + error={touched ? error : undefined} + > + + ); }); diff --git a/frontend/src/metabase/core/components/FormRadio/FormRadio.unit.spec.tsx b/frontend/src/metabase/core/components/FormRadio/FormRadio.unit.spec.tsx index 2ec7da20957c..c911b52629b8 100644 --- a/frontend/src/metabase/core/components/FormRadio/FormRadio.unit.spec.tsx +++ b/frontend/src/metabase/core/components/FormRadio/FormRadio.unit.spec.tsx @@ -3,11 +3,10 @@ import { Form, Formik } from "formik"; import * as Yup from "yup"; import { render, screen, waitFor } from "@testing-library/react"; import userEvent from "@testing-library/user-event"; -import FormField from "metabase/core/components/FormField"; import FormRadio from "./FormRadio"; -const TEST_SCHEMA = Yup.object().shape({ - value: Yup.string().notOneOf(["Bar"]), +const TEST_SCHEMA = Yup.object({ + value: Yup.string().notOneOf(["bar"], "error"), }); const TEST_OPTIONS = [ @@ -29,9 +28,7 @@ const TestFormRadio = ({ initialValue, onSubmit }: TestFormRadioProps) => { onSubmit={onSubmit} >
- - - + @@ -47,23 +44,26 @@ describe("FormRadio", () => { expect(screen.getByRole("radio", { name: "Line" })).toBeChecked(); }); - it("should propagate the changed value to the form", () => { + it("should propagate the changed value to the form", async () => { const onSubmit = jest.fn(); render(); userEvent.click(screen.getByRole("radio", { name: "Line" })); userEvent.click(screen.getByText("Submit")); - waitFor(() => expect(onSubmit).toHaveBeenCalledWith({ value: "line" })); + await waitFor(() => { + const values = { value: "line" }; + expect(onSubmit).toHaveBeenCalledWith(values, expect.anything()); + }); }); - it("should be validated on blur", () => { + it("should be validated on blur", async () => { const onSubmit = jest.fn(); render(); userEvent.click(screen.getByRole("radio", { name: "Bar" })); userEvent.tab(); - waitFor(() => expect(screen.getByText("Label: error")).toBeInTheDocument()); + expect(await screen.findByText(": error")).toBeInTheDocument(); }); }); diff --git a/frontend/src/metabase/core/components/FormSelect/FormSelect.tsx b/frontend/src/metabase/core/components/FormSelect/FormSelect.tsx index bd0d2763af34..d4d7721aeca2 100644 --- a/frontend/src/metabase/core/components/FormSelect/FormSelect.tsx +++ b/frontend/src/metabase/core/components/FormSelect/FormSelect.tsx @@ -1,31 +1,54 @@ -import React, { useMemo } from "react"; +import React, { forwardRef, ReactNode, Ref, useMemo } from "react"; import { useField } from "formik"; +import { useUniqueId } from "metabase/hooks/use-unique-id"; import Select, { SelectOption, SelectProps, } from "metabase/core/components/Select"; +import FormField from "metabase/core/components/FormField"; export interface FormSelectProps> extends Omit, "value" | "onChange"> { name: string; + title?: string; + description?: ReactNode; } -function FormSelect>({ - name, - ...props -}: FormSelectProps) { - const [{ value, onChange, onBlur }] = useField(name); - const buttonProps = useMemo(() => ({ id: name, onBlur }), [name, onBlur]); +const FormSelect = forwardRef(function FormSelect< + TValue, + TOption = SelectOption, +>( + { + name, + className, + title, + description, + ...props + }: FormSelectProps, + ref: Ref, +) { + const id = useUniqueId(); + const [{ value, onChange, onBlur }, { error, touched }] = useField(name); + const buttonProps = useMemo(() => ({ id, onBlur }), [id, onBlur]); return ( - + ); -} +}); export default FormSelect; diff --git a/frontend/src/metabase/core/components/FormSelect/FormSelect.unit.spec.tsx b/frontend/src/metabase/core/components/FormSelect/FormSelect.unit.spec.tsx index 68363fdbe359..5065da1713bd 100644 --- a/frontend/src/metabase/core/components/FormSelect/FormSelect.unit.spec.tsx +++ b/frontend/src/metabase/core/components/FormSelect/FormSelect.unit.spec.tsx @@ -3,11 +3,10 @@ import { Form, Formik } from "formik"; import * as Yup from "yup"; import { render, screen, waitFor } from "@testing-library/react"; import userEvent from "@testing-library/user-event"; -import FormField from "metabase/core/components/FormField"; import FormSelect from "./FormSelect"; -const TEST_SCHEMA = Yup.object().shape({ - value: Yup.string().notOneOf(["Bar"]), +const TEST_SCHEMA = Yup.object({ + value: Yup.string().notOneOf(["bar"], "error"), }); const TEST_OPTIONS = [ @@ -29,13 +28,12 @@ const TestFormSelect = ({ initialValue, onSubmit }: TestFormSelectProps) => { onSubmit={onSubmit} >
- - - + @@ -51,7 +49,7 @@ describe("FormSelect", () => { expect(screen.getByText("Line")).toBeInTheDocument(); }); - it("should propagate the changed value to the form", () => { + it("should propagate the changed value to the form", async () => { const onSubmit = jest.fn(); render(); @@ -59,7 +57,10 @@ describe("FormSelect", () => { userEvent.click(screen.getByText("Line")); userEvent.click(screen.getByText("Submit")); - waitFor(() => expect(onSubmit).toHaveBeenCalledWith({ value: "line" })); + await waitFor(() => { + const values = { value: "line" }; + expect(onSubmit).toHaveBeenCalledWith(values, expect.anything()); + }); }); it("should be referenced by the label", () => { @@ -70,14 +71,14 @@ describe("FormSelect", () => { expect(screen.getByLabelText("Label")).toBeInTheDocument(); }); - it("should be validated on blur", () => { + it("should be validated on blur", async () => { const onSubmit = jest.fn(); render(); userEvent.click(screen.getByText("Line")); userEvent.click(screen.getByText("Bar")); - userEvent.tab(); + userEvent.click(screen.getByText("Submit")); - waitFor(() => expect(screen.getByText("Label: error")).toBeInTheDocument()); + expect(await screen.findByText(": error")).toBeInTheDocument(); }); }); diff --git a/frontend/src/metabase/core/components/FormSubmitButton/FormSubmitButton.tsx b/frontend/src/metabase/core/components/FormSubmitButton/FormSubmitButton.tsx index 6878df90e059..05a1bff5f6eb 100644 --- a/frontend/src/metabase/core/components/FormSubmitButton/FormSubmitButton.tsx +++ b/frontend/src/metabase/core/components/FormSubmitButton/FormSubmitButton.tsx @@ -1,59 +1,56 @@ import React, { forwardRef, Ref } from "react"; -import { useFormikContext } from "formik"; import { t } from "ttag"; import Button, { ButtonProps } from "metabase/core/components/Button"; -import { FormStatus } from "metabase/core/hooks/use-form-state"; -import useFormStatus from "metabase/core/hooks/use-form-status"; +import useFormSubmitButton from "metabase/core/hooks/use-form-submit-button"; +import { FormStatus } from "metabase/core/context/FormContext"; export interface FormSubmitButtonProps extends Omit { - normalText?: string; - activeText?: string; - successText?: string; - failedText?: string; + title?: string; + activeTitle?: string; + successTitle?: string; + failedTitle?: string; } const FormSubmitButton = forwardRef(function FormSubmitButton( - { disabled, ...props }: FormSubmitButtonProps, + { primary, disabled, ...props }: FormSubmitButtonProps, ref: Ref, ) { - const { isValid, isSubmitting } = useFormikContext(); - const status = useFormStatus(); - const submitText = getSubmitButtonText(status, props); - const isEnabled = isValid && !isSubmitting && !disabled; + const { status, isDisabled } = useFormSubmitButton({ isDisabled: disabled }); + const submitTitle = getSubmitButtonTitle(status, props); return ( ); }); -const getSubmitButtonText = ( +const getSubmitButtonTitle = ( status: FormStatus | undefined, { - normalText = t`Submit`, - activeText = normalText, - successText = t`Success`, - failedText = t`Failed`, + title = t`Submit`, + activeTitle = title, + successTitle = t`Success`, + failedTitle = t`Failed`, }: FormSubmitButtonProps, ) => { switch (status) { case "pending": - return activeText; + return activeTitle; case "fulfilled": - return successText; + return successTitle; case "rejected": - return failedText; + return failedTitle; default: - return normalText; + return title; } }; diff --git a/frontend/src/metabase/core/components/FormTextArea/FormTextArea.tsx b/frontend/src/metabase/core/components/FormTextArea/FormTextArea.tsx new file mode 100644 index 000000000000..bb1bb4b5bf48 --- /dev/null +++ b/frontend/src/metabase/core/components/FormTextArea/FormTextArea.tsx @@ -0,0 +1,76 @@ +import React, { + ChangeEvent, + forwardRef, + ReactNode, + Ref, + useCallback, +} from "react"; +import { useField } from "formik"; +import { useUniqueId } from "metabase/hooks/use-unique-id"; +import TextArea, { TextAreaProps } from "metabase/core/components/TextArea"; +import FormField from "metabase/core/components/FormField"; + +export interface FormTextAreaProps + extends Omit< + TextAreaProps, + "value" | "error" | "fullWidth" | "onChange" | "onBlur" + > { + name: string; + title?: string; + description?: ReactNode; + nullable?: boolean; + infoLabel?: string; + infoTooltip?: string; +} + +const FormTextArea = forwardRef(function FormTextArea( + { + name, + className, + style, + title, + description, + nullable, + infoLabel, + infoTooltip, + ...props + }: FormTextAreaProps, + ref: Ref, +) { + const id = useUniqueId(); + const [{ value, onBlur }, { error, touched }, { setValue }] = useField(name); + + const handleChange = useCallback( + ({ target: { value } }: ChangeEvent) => { + setValue(value === "" && nullable ? null : value); + }, + [nullable, setValue], + ); + + return ( + +