Check out this Statistical Test WhizzML script "Statistical Tests"

Statistical tests | BigML.com

FREE

Statistical Tests

Details:

Created	Thu, 8 Oct 2026 15:03:18 +0000
Published	Thu, 8 Oct 2026 17:25:42 +0000

Description:

Although BigML offers Statistical Tests through its API, they are not offered in the Dashboard.

In this package you'll find a script that obtains all the statistical tests for a given dataset and creates a report dataset with the results. It also returns a map with aggregated metrics across all the numeric fields.

One interesting thing to do with the output dataset is using the Scatter Plot view to visualize the metrics for all the fields in your original dataset.

Find more info in readme

Tags:

BigMLer BigMLer_ThuOct0826_170317

URL:

https://bigml.com/user/whizzml/gallery/script/6ac7b0b6c31cdddafbef9045

Script FREE

Source code

;; statistical-tests
;;
;; given a dataset
;; performs several statistical tests and returns their results,
;; along with a summary of the average values and standard
;; deviations across all the fields


;; From a map with all the fields and a field id,
;; returns its corresponding field name
(define (field-name fields fid)
  (fields [fid "name"]))


;; Transforms a list of maps into a single map
(define (flatten-map list-of-maps)
  (reduce merge {} list-of-maps))


;; From a list of maps, obtains the first map with a specified
;; key-value pair
(define (first-match-key list-of-maps key value)
  (head (filter (lambda (n) (= (n key) value)) list-of-maps)))

;; From the results of a test, returns the specific ones for
;; a field. It removes unnecessary fields
(define (analyze-test-field fields test-results fid)
  (let (test-field (test-results fid)
        fname (field-name fields fid))
    {fname (dissoc test-field "significant")}))

;; From the complete set of statistical tests, returns a map with all
;; the metrics for one of them removing unnecesary keys and
;; transforming field ids into field names
(define (analyze-test type subtype test)
  (let (test-results ((first-match-key (test type) "name" subtype) "result"))
    (flatten-map (map (partial analyze-test-field (test "fields") test-results)
                      (keys test-results)))))

;; From the results of Benford test, returns the specific ones for
;; a field. It removes unnecessary fields
(define (analyze-benford-field fields benford fid)
  (let (benford-field (benford fid)
        fname (field-name fields fid))
    {fname {"chi_square" (benford-field ["chi_square" "chi_square_value"])
            "p_value" (benford-field ["chi_square" "p_value"])
            "distribution" (benford-field "distribution")
            "d_statistic" (benford-field ["cho_gaines" "d_statistic"])}}))

;; Obtains all the metrics from the Benford test within a map.
;; This specific test can't be analyzed with the function
;; analyze-test as the others
(define (analyze-benford test)
  (let (benford (get (head (test "fraud")) "result"))
    (flatten-map (map (partial analyze-benford-field (test "fields") benford)
                      (keys benford)))))

;; Performs all the statistical tests for the configured dataset-id
;; using the ad-sample-size and ad-seed values set by the user
(define (statistical-tests dataset-id ad-sample-size ad-seed)
  (let (seed (if (> (count ad-seed) 0) ad-seed "bigml")
        tests (get (fetch (create-and-wait-statisticaltest
                           dataset-id
                           {"ad_sample_size" ad-sample-size
                            "ad_seed" seed}))
                   "statistical_tests"))))

;; From a given statistical test, computes average and standard
;; deviation values for a single metric across all the fields.
(define (summary-metric test metric)
  (let (metric-values (map (lambda (field) (get (test field) metric))
                           (keys test)))
    {"average" (mean metric-values)
     "stdev" (stdev metric-values)}))

;; From a given statistical test, computes average and standard
;; deviation values for all the metrics across all the fields.
(define (summary-all-metrics test test-name metrics)
  {test-name
   (flatten-map (map (lambda (m) {m (summary-metric test m)}) metrics))})

;; Remove not needed parameters from tests
(define (clean-test test)
  (dissoc test "distribution" "significant"))

;; Obtains all the metrics for a specific field and join all of the
;; in a single string sepparated with comas
(define (field-tests-data field)
  (str field  ","
       (join ","  (flatten (map (lambda (test)
                                  (values (clean-test (test field))))
                                [benford anderson jarque-bera z-score grubbs])))))

;; Obtains the names of all the metrics
(define (field-tests-keys field)
  (join "," (flatten (map (lambda (test name)
                            (map (lambda (n) (str name "-" n))
                                 (keys (clean-test (test field)))))
                          [benford anderson jarque-bera z-score grubbs]
                          ["benford" "anderson" "jarque" "z-score" "grubbs"]))))

;; Creates a Source with all the metrics from all the statistical tests
(define (create-source-data benford)
  (let (fields (keys benford)
        header (str "field," (field-tests-keys (head fields))))
    (join "\n" (cons header (map field-tests-data fields)))))

;; Creates the final Dataset with the tests results
(define (create-output-dataset dataset-id)
  (create-dataset (create-source {"data" (create-source-data benford)
                                  "name" (str (resource-name dataset-id)
                                              "-StatisticalTests")})))


(define output (statistical-tests dataset-id ad-sample-size ad-seed))
(define benford (analyze-benford output))
(define anderson (analyze-test "normality" "anderson_darling" output))
(define jarque-bera (analyze-test "normality" "jarque_bera" output))
(define z-score (analyze-test "normality" "z_score" output))
(define grubbs (analyze-test "outliers" "grubbs" output))
(define summary
  (flatten-map (map summary-all-metrics
                    [benford anderson jarque-bera z-score grubbs]
                    ["benford" "anderson" "jarque-bera" "z-score" "grubs"]
                    [["chi_square" "p_value" "d_statistic"]
                     ["p_value"]
                     ["p_value"]
                     ["expected_max_z" "max_z"]
                     ["p_value"]])))
(define tests-results-dataset (create-output-dataset dataset-id))

Description

Inputs

Outputs

License: Creative Commons Zero V1.0
Script FREE

Statistical tests | BigML.com

Sending Request...

COMPANY

PRODUCT

BUSINESS

TRAINING

GALLERY

License

Embed this Script in your web site

COMPANY

PRODUCT

BUSINESS

TRAINING

GALLERY