Ensemble's k-fold cross-validation

Ensemble's k-fold cross-validation | BigML.com
FREE
Details:
Created	Wed, 27 Jul 2016 06:59:52 +0000
Published	Wed, 27 Jul 2016 08:21:45 +0000
Description:
The objective of this script is to perform a k-fold cross validation of an ensemble built from a dataset. The algorithm:
Divides the dataset in k parts.
Holds out the data in one of the parts and builds an ensemble with the rest of data.
Evaluates the ensemble with the hold out data.
The second and third steps are repeated with each of the k parts, so that k evaluations are generated.
Finally, the evaluation metrics are averaged to provide the cross-validation metrics.
The output of the script will be an evaluation ID. This evaluation is a cross-validation, meaning that its metrics are averages of the k evaluations created in the cross-validation process.
Tags:
Evaluations Ensembles Cross-validation
URL:
https://bigml.com/user/whizzml/gallery/script/57985be8af447f1f4c000724
Script FREE
Source code
;; This code will eventually be defined as a library.

(define MODEL_OPTIONS ["balance_objective"
                       "missing_splits"
                       "pruning"
                       "weight_field"
                       "objective_weights"
                       "node_threshold"])
(define ENSEMBLE_OPTIONS (concat MODEL_OPTIONS
                                 ["sample_rate"
                                  "replacement"
                                  "randomize"
                                  "number_of_models"
                                  "seed"]))
(define LOGISTIC_OPTIONS ["balance_fields"
                          "bias"
                          "c"
                          "missing_numerics"
                          "default_numeric_value"
                          "eps"
                          "field_codings"
                          "normalize"
                          "regularization"
                          "seed"])
(define EVALUATION_OPTIONS ["sample_rate"
                            "out_of_bag"
                            "range"
                            "replacement"
                            "ordering"
                            "seed"
                            "missing_strategy"
                            "combiner"])
;; cross-validation
;;
;; creates k-fold cross-validation for a dataset
;; Inputs:
;;   dataset-id: (string) Dataset ID
;;   k-folds: (integer) Number of folds
;;   model-options: (map) Options to use in model/ensemble
;;   evaluation-options: (map) Options to use in evaluation creation
;;
;; Output: (map) Average of evaluations results
;;
;; Raises:
;;  101: The dataset-id argument is not a string
;;  102: The dataset-id is not a valid dataset ID
;;  103: The k-folds argument is not an integer
;;  104: The k-folds argument is not >= 2
;;  106: The objective field ID is not in the selectable IDs list
;;
(define (cross-validation dataset-id
                          k-folds
                          objective-id
                          model-options
                          evaluation-options)
  (check-resource-id dataset-id "dataset")
  (check-integer k-folds 2 false)
  (let (dataset (fetch dataset-id)
        dataset-name (dataset "name" false))
    (check-dataset-objective-id objective-id dataset)
    (let (k-fold-datasets (create-k-folds dataset-id k-folds)
         objective-name (get-objective-name dataset objective-id)
         evaluations (create-k-evaluations k-fold-datasets
                                           objective-name
                                           dataset-name
                                           model-options
                                           evaluation-options))
     (create-and-wait-evaluation {"evaluations" evaluations}))))

;; check-resource-id
;;
;; Validates that the argument is a resource ID and its type. Raises an error
;; if otherwise.
;;
;; Inputs:
;;   resource-id: (string) Resource ID
;;   type: (string) Type of resource
;;
;; Output: (string) Checked resource ID
(define (check-resource-id resource-id type)
  (when (not (string? resource-id))
    (raise {"message" (str "Resource ID string expected. Found "
                           resource-id " instead.")
            "code" 101}))
  (when (not (= (resource-type resource-id) type))
    (raise {"message" (str "Failed to find a correct " type " ID.")
            "code" 102}))
  resource-id)


;; check-integer
;;
;; Validates that the argument is an integer. Raises error if otherwise.
;;
;; Inputs:
;;  value: (number) Integer to be checked
;;  minimum: (number) Minimum value (false if not set)
;;  maximum: (number) Maximum value (false if not set)
;;
;; Output: (number) Checked integer
(define (check-integer value minimum maximum)
  (when (not (integer? value))
    (raise {"message" (str "Integer value expected. Found " value " instead.")
            "code" 103}))
  (when (and minimum (< value minimum))
    (raise {"message" (str "Minimum accepted value is " minimum ". " value
                           " found.")
            "code" 104}))
  (when (and maximum (> value maximum))
    (raise {"message" (str "Maximum accepted value is " maximum ". " value
                           " found.")
            "code" 105}))
  value)

;; choosable-objective-ids
;;
;; List of IDs of the fields in the dataset that can be chosen as objective
;; field.
;;
;; Inputs:
;;  fields: (map) Fields structure
;; Output: (list) list of field IDs
(define (choosable-objective-ids fields)
  (let (field-val (lambda (fid k) (fields [fid k] false))
        objective-types ["categorical", "numeric"]
        pref? (lambda (k) (field-val k "preferred"))
        pred? (lambda (k) (member? (field-val k "optype") objective-types)))
    (filter (lambda (x) (and (pref? x) (pred? x))) (keys fields))))


;; check-dataset-objective-id
;;
;; Validates that the argument is a valid objective id in the reference
;; dataset.
;;
;; Inputs:
;;  objective-id: (string) ID of the objective field
;;  dataset: (map) Dataset resource information
;;
;; Output: (string) Checked objective field ID
(define (check-dataset-objective-id objective-id dataset)
  (let (fields (dataset "fields" {})
        objective-ids (choosable-objective-ids fields))
    (when (not (member? objective-id objective-ids))
      (raise {"message" (str "Failed to find the objective ID in the dataset"
                             " choosable fields.")
              "code" 106}))))

;; get-objective-name
;;
;; Returns the name of the field used as objective field
;;
;; Inputs:
;;  dataset: (map) Dataset resource info
;;  objective-id: (string) ID of the objective field
;;
;; Outputs: (string) Name of the objective field

(define (get-objective-name dataset objective-id)
  (dataset ["fields" objective-id "name"] false))


;; create-k-folds
;;
;; creating k-fold splits from a dataset
;;
;; Inputs:
;;   dataset-id: (string) Dataset ID
;;   k-folds: (integer) Number of folds
;;
;; Output: (list) List of dataset IDs
;;
(define (create-k-folds dataset-id k-folds)
  (let (k-fold-fn (lambda (x)
                    (create-dataset {"origin_dataset" dataset-id
                                     "row_offset" x
                                     "row_step" k-folds
                                     "new_fields" [{"name" "k_fold"
                                                    "field" (str x)}]}))
        dataset-ids (map k-fold-fn (range 0 k-folds)))
    (wait* dataset-ids)))

;; pair-k-folds
;;
;; Builds a list of pairs of hold-out and complementary datasets for all
;; the k-fold dataset IDs.
;;
;; Inputs:
;;   dataset-ids: (list) List of the k-fold dataset IDs
;;
;; Output: (list) List of pairs [hold-out dataset, multidataset with the rest]
;;
(define (pair-k-folds dataset-ids)
  (map (lambda(x)
         [(nth dataset-ids x)
          (concat (take x dataset-ids)
          (drop (+ x 1) dataset-ids))])
       (range 0 (count dataset-ids))))


;; select-map-keys
;;
;; Filters the keys in a map, keeping only the ones that appear in the list.
;;
;; Inputs:
;;   map: (map) Key, value maps
;;   keys-list: (list) List of keys to be kept in the map
;; Output: (map) filtered map with only the keys in the keys-list
;;
(define (select-map-keys a-map keys-list)
  (reduce (lambda (x y) (let (value (a-map y false))
                          (cond value (assoc x y value) x)))
          {}
          keys-list))

;; create-k-models
;;
;; Creates the models for a set of k-fold datasets
;;
;; Inputs:
;;   type: (string) type of model (model or ensemble)
;;   multidatasets: (list) List of lists of datset IDs once a k-fold is
;;                         excluded
;;   objective-name: (string) name of the objective field
;;   model-options: (map) Options for the model or ensemble
;;
;; Output: (list) model IDs
;;
(define (create-k-models type multidatasets objective-name model-options)
  (let (models (map (lambda (x)
                      (create type
                              (merge {"datasets" x
                                      "objective_field" objective-name}
                                     model-options)))
                     multidatasets))
    (wait* models)))

;; create-k-evaluations
;;
;; Creates the models/ensembles and evaluations for a set of k-fold datasets
;;
;; Inputs:
;;   dataset-ids: (list) List of the k-fold dataset IDs
;;   objective-name: (string) Objective field name
;;   dataset-name: (string) Name of the origin dataset
;;   model-options: (map) Options used to build the models/ensembles
;;   evaluation-options: (map) Options used to build evaluations
;;
;; Output: (list) List of evaluation IDs
;;
(define (create-k-evaluations dataset-ids
                              objective-name
                              dataset-name
                              model-options
                              evaluation-options)
  (let (number-of-models (model-options "number_of_models" 1)
        regularization (model-options "regularization" false)
        k-fold-pairs (pair-k-folds dataset-ids)
        ensemble-options (select-map-keys model-options ENSEMBLE_OPTIONS)
        model-options (select-map-keys model-options MODEL_OPTIONS)
        logistic-options (select-map-keys model-options LOGISTIC_OPTIONS)
        evaluation-options (select-map-keys evaluation-options
                                            EVALUATION_OPTIONS)
        type (if (> number-of-models 1) "ensemble" "model")
        multidatasets (map last k-fold-pairs)
        models (cond (> number-of-models 1)
                     (create-k-models type
                                      multidatasets
                                      objective-name
                                      ensemble-options)
                     regularization
                     (create-k-models type
                                      multidatasets
                                      objective-name
                                      logistic-options)
                     (create-k-models type
                                      multidatasets
                                      objective-name
                                      model-options))
        evaluations (iterate (es []
                              id dataset-ids
                              mid models
                              idx (range 1 (+ 1 (count dataset-ids))))
                      (let (name (str idx "-fold evaluation " dataset-name)
                            opts (assoc evaluation-options "name" name))
                       (append es (create-evaluation id mid opts)))))
    (wait* evaluations)))

;; Script

;;get-model-options
;;
;; maps the options to be used in models
;; Inputs:
;;   missing-splits: (boolen) Sets the missing_splits flag
;;   stat-pruning: (boolean) Sets the statistical pruning flag
;;   balance-objective: (boolean) Sets the balance_objective flag
;;   weight-field: (string) ID of the field to be used as weight (weight_field)
;;   objetive-weights: (list) List of values to be used as objective_weights
;;   node-threshold: (integer) Maximum number of nodes in the model
;; Output: (map) options map
;;
(define (get-model-options missing-splits
                           stat-pruning
                           balance-objective
                           weight-field
                           objetive-weights
                           node-threshold)
  (let (options {}
    options (assoc options "missing_splits" missing-splits)
    options (assoc options "stat_pruning" stat-pruning)
    options (assoc options "balance_objective" balance-objective)
    options (if (not (empty? weight-field))
                (assoc options "weight_field" weight-field)
                options)
    options (if (not (empty? objective-weights))
                (assoc options "objective_weights" objective-weights)
                options)
    options (if (not (= node-threshold -1))
                (assoc options "node_threshold" node-threshold)
                options))
    options))

;;get-ensemble-options
;;
;; maps the options to be used in esembles
;; Inputs:
;;   number-of-models: (integer) Number of models in the ensemble
;;   missing-splits: (boolean) Sets the missing_splits flag
;;   stat-pruning: (boolean) Sets the pruning flag
;;   balance-objective: (boolean) Sets the balance_objective flag
;;   weight-field: (string) ID of the field to be used as weight (weight_field)
;;   objective-weights: (list) List of values to be used as objective_weights
;;   node-threshold: (integer) Maximum number of nodes in the model
;;   sample-rate: (float) Percentage of instances used as sample
;;   replacement: (boolean) Sets the replacement flag
;;   randomize: (boolean) Sets the randomize flag
;;   seed: (string) Seed used in random samplings
;; Output: (map) options map
;;
(define (get-ensemble-options number-of-models
                              missing-splits
                              stat-pruning
                              balance-objective
                              weight-field
                              objetive-weights
                              node-threshold
                              sample-rate
                              replacement
                              randomize
                              seed)
  (let (options (get-model-options missing-splits
                                   stat-pruning
                                   balance-objective
                                   weight-field
                                   objective-weights
                                   node-threshold)
        options (if (> number-of-models 1)
                    (assoc options "number_of_models" number-of-models)
                    (assoc options "number_of_models" 10))
        options (merge options {"sample_rate" sample-rate
                                "replacement" replacement
                                "randomize" randomize}))
    (if (empty? seed)
        options
        (assoc options "seed" seed))))


;; ensemble-cross-validation
;;
;; creates k-fold cross-validation for a dataset using ensembles
;; Inputs:
;;   dataset-id: (dataset-id) Dataset ID
;;   k-folds: (integer) Number of folds
;;   objective-id: (string) ID of the objective field
;;   number-of-models: (integer) Number of models in the ensemble
;;   missing-splits: (boolean) sets the missing_splits flag
;;   stat-pruning: (boolean) Sets the statistical pruning flag
;;   balance-objective: (boolean) sets the balance_objective flag
;;   weight-field: (string) ID of the field to be used as weight_field
;;   objective-weights: (list) List of objective_weights
;;   node-threshold: (integer) Maximum number of nodes in the model
;;   sample-rate: (float) Percentage of data used in sampling
;;   replacement: (boolean) Sets the replacement flag
;;   randomize: (boolean) Sets the randomize flag
;;   seed: (string) Seed used in random samplings
;;
;; Output: (evaluation-id) Average of evaluations results
;;
(define (ensemble-cross-validation dataset-id
                                   k-folds
                                   objective-id
                                   number-of-models
                                   missing-splits
                                   stat-pruning
                                   balance-objective
                                   weight-field
                                   objective-weights
                                   node-threshold
                                   sample-rate
                                   replacement
                                   randomize
                                   seed)
  (let (options (get-ensemble-options number-of-models
                                      missing-splits
                                      stat-pruning
                                      balance-objective
                                      weight-field
                                      objective-weights
                                      node-threshold
                                      sample-rate
                                      replacement
                                      randomize
                                      seed)
        objective-id (if (empty? objective-id)
                         (dataset-get-objective-id dataset-id)
                         objective-id))
    (cross-validation dataset-id k-folds objective-id options {})))


(define cross-validation-output (ensemble-cross-validation
                                 dataset-id
                                 k-folds
                                 objective-id
                                 number-of-models
                                 missing-splits
                                 stat-pruning
                                 balance-objective
                                 weight-field
                                 objective-weights
                                 node-threshold
                                 sample-rate
                                 replacement
                                 randomize
                                 seed))
Description
Inputs
Outputs
License: Creative Commons Zero V1.0
Script FREE
Ensemble's k-fold cross-validation | BigML.com
Sending Request...
COMPANY

PRODUCT

BUSINESS

TRAINING

GALLERY

License

Embed this Script in your web site

COMPANY

PRODUCT

BUSINESS

TRAINING

GALLERY
