Logistic regression's k-fold cross-validation

Logistic regression's k-fold cross-validation | BigML.com
FREE
Details:
Created	Tue, 6 Mar 2018 18:05:48 +0000
Published	Tue, 20 Mar 2018 14:19:53 +0000
Description:
The objective of this script is to perform a k-fold cross validation of a logistic regression built from a dataset. The algorithm:
Divides the dataset in k parts
Holds out the data in one of the parts and builds a logistic regression with the rest of data
Evaluates the logistic regression with the hold out data
The second and third steps are repeated with each of the k parts, so that k evaluations are generated
Finally, the evaluation metrics are averaged to provide the cross-validation metrics.
The output of the script will be an evaluation ID. This evaluation is a cross-validation, meaning that its metrics are averages of the k evaluations created in the cross-validation process.
For more information, please see the readme.
Tags:
URL:
https://bigml.com/user/whizzml/gallery/script/5a9ed87ceba31d3b3b0002ba
Script FREE
Source code
;; This code will eventually be defined as a library.

(define LOGISTIC_OPTIONS ["balance_fields"
                          "bias"
                          "c"
                          "missing_numerics"
                          "default_numeric_value"
                          "eps"
                          "field_codings"
                          "normalize"
                          "regularization"
                          "seed"])
(define EVALUATION_OPTIONS ["sample_rate"
                            "out_of_bag"
                            "range"
                            "replacement"
                            "ordering"
                            "seed"
                            "missing_strategy"
                            "combiner"])
;; cross-validation
;;
;; creates k-fold cross-validation for a dataset
;; Inputs:
;;   dataset-id: (string) Dataset ID
;;   k-folds: (integer) Number of folds
;;   model-options: (map) Options to use in model/ensemble
;;   evaluation-options: (map) Options to use in evaluation creation
;;   delete-resources: (boolean) Whether to delete all intermediate resources
;;
;; Output: (map) Average of evaluations results
;;
;; Raises:
;;  101: The dataset-id argument is not a string
;;  102: The dataset-id is not a valid dataset ID
;;  103: The k-folds argument is not an integer
;;  104: The k-folds argument is not >= 2
;;  106: The objective field ID is not in the selectable IDs list
;;
(define (cross-validation dataset-id
                          k-folds
                          objective-id
                          model-options
                          evaluation-options
                          delete-resources)
  (check-resource-id dataset-id "dataset")
  (check-integer k-folds 2 false)
  (let (dataset (fetch dataset-id)
        dataset-name (dataset "name" false))
    (check-dataset-objective-id objective-id dataset)
    (let (k-fold-datasets (create-k-folds dataset-id k-folds)
         objective-name (get-objective-name dataset objective-id)
         evaluations (create-k-evaluations k-fold-datasets
                                           objective-name
                                           dataset-name
                                           model-options
                                           evaluation-options
                                           delete-resources)
         evaluations-average (create-and-wait-evaluation {"evaluations" evaluations}))
      (when delete-resources
        (map safe-delete (concat k-fold-datasets evaluations)))
      evaluations-average)))

;; check-resource-id
;;
;; Validates that the argument is a resource ID and its type. Raises an error
;; if otherwise.
;;
;; Inputs:
;;   resource-id: (string) Resource ID
;;   type: (string) Type of resource
;;
;; Output: (string) Checked resource ID
(define (check-resource-id resource-id type)
  (when (not (string? resource-id))
    (raise {"message" (str "Resource ID string expected. Found "
                           resource-id " instead.")
            "code" 101}))
  (when (not (= (resource-type resource-id) type))
    (raise {"message" (str "Failed to find a correct " type " ID.")
            "code" 102}))
  resource-id)


;; check-integer
;;
;; Validates that the argument is an integer. Raises error if otherwise.
;;
;; Inputs:
;;  value: (number) Integer to be checked
;;  minimum: (number) Minimum value (false if not set)
;;  maximum: (number) Maximum value (false if not set)
;;
;; Output: (number) Checked integer
(define (check-integer value minimum maximum)
  (when (not (integer? value))
    (raise {"message" (str "Integer value expected. Found " value " instead.")
            "code" 103}))
  (when (and minimum (< value minimum))
    (raise {"message" (str "Minimum accepted value is " minimum ". " value
                           " found.")
            "code" 104}))
  (when (and maximum (> value maximum))
    (raise {"message" (str "Maximum accepted value is " maximum ". " value
                           " found.")
            "code" 105}))
  value)

;; choosable-objective-ids
;;
;; List of IDs of the fields in the dataset that can be chosen as objective
;; field.
;;
;; Inputs:
;;  fields: (map) Fields structure
;; Output: (list) list of field IDs
(define (choosable-objective-ids fields)
  (let (field-val (lambda (fid k) (fields [fid k] false))
        objective-types ["categorical", "numeric"]
        pref? (lambda (k) (field-val k "preferred"))
        pred? (lambda (k) (member? (field-val k "optype") objective-types)))
    (filter (lambda (x) (and (pref? x) (pred? x))) (keys fields))))


;; check-dataset-objective-id
;;
;; Validates that the argument is a valid objective id in the reference
;; dataset.
;;
;; Inputs:
;;  objective-id: (string) ID of the objective field
;;  dataset: (map) Dataset resource information
;;
;; Output: (string) Checked objective field ID
(define (check-dataset-objective-id objective-id dataset)
  (let (fields (dataset "fields" {})
        objective-ids (choosable-objective-ids fields))
    (when (not (member? objective-id objective-ids))
      (raise {"message" (str "Failed to find the objective ID in the dataset"
                             " choosable fields.")
              "code" 106}))))

;; get-objective-name
;;
;; Returns the name of the field used as objective field
;;
;; Inputs:
;;  dataset: (map) Dataset resource info
;;  objective-id: (string) ID of the objective field
;;
;; Outputs: (string) Name of the objective field

(define (get-objective-name dataset objective-id)
  (let (fields (dataset "fields" {}))
    (fields [objective-id "name"] false)))


;; create-k-folds
;;
;; creating k-fold splits from a dataset
;;
;; Inputs:
;;   dataset-id: (string) Dataset ID
;;   k-folds: (integer) Number of folds
;;
;; Output: (list) List of dataset IDs
;;
(define (create-k-folds dataset-id k-folds)
  (let (k-fold-fn (lambda (x)
                    (create-dataset {"origin_dataset" dataset-id
                                     "row_offset" x
                                     "row_step" k-folds
                                     "new_fields" [{"name" "k_fold"
                                                    "field" (str x)}]}))
        dataset-ids (map k-fold-fn (range 0 k-folds)))
    (wait* dataset-ids)))

;; pair-k-folds
;;
;; Builds a list of pairs of hold-out and complementary datasets for all
;; the k-fold dataset IDs.
;;
;; Inputs:
;;   dataset-ids: (list) List of the k-fold dataset IDs
;;
;; Output: (list) List of pairs [hold-out dataset, multidataset with the rest]
;;
(define (pair-k-folds dataset-ids)
  (map (lambda(x)
         [(nth dataset-ids x)
          (concat (take x dataset-ids)
          (drop (+ x 1) dataset-ids))])
       (range 0 (count dataset-ids))))


;; select-map-keys
;;
;; Filters the keys in a map, keeping only the ones that appear in the list.
;;
;; Inputs:
;;   map: (map) Key, value maps
;;   keys-list: (list) List of keys to be kept in the map
;; Output: (map) filtered map with only the keys in the keys-list
;;
(define (select-map-keys a-map keys-list)
  (reduce (lambda (x y) (let (value (a-map y false))
                          (cond value (assoc x y value) x)))
          {}
          keys-list))

;; create-k-models
;;
;; Creates the models for a set of k-fold datasets
;;
;; Inputs:
;;   type: (string) type of model (model or ensemble)
;;   multidatasets: (list) List of lists of datset IDs once a k-fold is
;;                         excluded
;;   objective-name: (string) name of the objective field
;;   model-options: (map) Options for the model or ensemble
;;
;; Output: (list) model IDs
;;
(define (create-k-models type multidatasets objective-name model-options)
  (let (models (map (lambda (x)
                      (create type
                              (merge {"datasets" x
                                      "objective_field" objective-name}
                                     model-options)))
                     multidatasets))
    (wait* models)))

;; create-k-evaluations
;;
;; Creates the models/ensembles and evaluations for a set of k-fold datasets
;;
;; Inputs:
;;   dataset-ids: (list) List of the k-fold dataset IDs
;;   objective-name: (string) Objective field name
;;   dataset-name: (string) Name of the origin dataset
;;   model-options: (map) Options used to build the models/ensembles
;;   evaluation-options: (map) Options used to build evaluations
;;   delete-resources: (boolean) Whether to delete all intermediate resources   
;;
;; Output: (list) List of evaluation IDs
;;
(define (create-k-evaluations dataset-ids
                              objective-name
                              dataset-name
                              model-options
                              evaluation-options
                              delete-resources)
  (let (number-of-models (model-options "number_of_models" 1)
        regularization (model-options "regularization" false)
        k-fold-pairs (pair-k-folds dataset-ids)
        logistic-options (select-map-keys model-options LOGISTIC_OPTIONS)
        evaluation-options (select-map-keys evaluation-options
                                            EVALUATION_OPTIONS)
        multidatasets (map last k-fold-pairs)
        models  (create-k-models "logisticregression"
                                  multidatasets
                                  objective-name
                                  logistic-options)
        evaluations (iterate (es []
                              id dataset-ids
                              mid models
                              idx (range 1 (+ 1 (count dataset-ids))))
                      (let (name (str "Evaluation tested with subset "
                                      idx
                                      " of " dataset-name)
                            opts (assoc evaluation-options "name" name))
                       (append es (create-evaluation id mid opts)))))
    (wait* evaluations)
    (when delete-resources
      (map safe-delete models))
    evaluations))

;; Script

;;get_logistic_options
;;
;; maps the options to be used in logistic regression
;; Inputs:
;;   balance-fields: (boolean) Whether to scale each numeric field
;;   bias: (boolean) Whether to include an intercept term
;;   c: (number) Step of the regularization step
;;   missing-numerics: (boolean) Sets the logistic regression missing_numerics
;;                               flag
;;   default-numeric-value: (string) List of values to be used as
;;                                   objective_weights
;;   eps: (number) Stopping criteria for solver
;;   field-codings: (list) Coding schemes for categorical fields: 'dummy',
;;                         'contrast', or 'other'
;;   normalize: (boolean) Whether to normalize feature vectors
;;   regularization: (string) Selects the norm to minimize when regularizing
;;                            the solution: 'l1' or 'l2'
;;   seed: (string) Seed used in random samplings
;; Output: (map) options map
;;
(define (get_logistic_options balance-fields
                              bias
                              c
                              missing-numerics
                              default-numeric-value
                              eps
                              field-codings
                              normalize
                              regularization
                              seed)
  (let (options {"balance_fields" balance-fields
                 "bias" bias
                 "c" c
                 "missing_numerics" missing-numerics
                 "eps" eps
                 "field_codings" field-codings
                 "normalize" normalize
                 "regularization" regularization}
        options (if (empty? seed)
                    options
                   (assoc options "seed" seed)))
    (if (empty? default-numeric-value)
        options
        (assoc options "default_numeric_value" default-numeric-value))))

;; safe-delete
;;
;; deletes resources ignoring errors
;; Inputs:
;;   id: (resource-id) resource to be deleted
;;
;; Output: (boolean) true if successful, false if not
;;   
(define (safe-delete id)
  (try (delete id)
       (catch e 
         (log-info (str "Error deleting resource " id " ignored"))
         false)))

;; logistic-cross-validation
;;
;; creates k-fold cross-validation for a dataset using logistic regressions
;; Inputs:
;;   dataset-id: (string) Dataset ID
;;   k-folds: (integer) Number of folds
;;   objective-id: (string) ID of the objective field
;;   balance-fields: (boolean) Whether to scale each numeric field
;;   bias: (boolean) Whether to include an intercept term
;;   c: (number) Step of the regularization step
;;   missing-numerics: (boolean) Sets the logistic regression missing_numerics
;;                               flag
;;   default-numeric-value: (string) List of values to be used as
;;                                   objective_weights
;;   eps: (number) Stopping criteria for solver
;;   field-codings: (list) Coding schemes for categorical fields: 'dummy',
;;                         'contrast', or 'other'
;;   normalize: (boolean) Whether to normalize feature vectors
;;   regularization: (string) Selects the norm to minimize when regularizing
;;                            the solution: 'l1' or 'l2'
;;   seed: (string) Seed used in random samplings
;;   delete-resources: (boolean) Whether to delete all intermediate resources
;;
;; Output: (map) Average of evaluations results
;;
(define (logistic-cross-validation dataset-id
                                   k-folds
                                   objective-id
                                   balance-fields
                                   bias
                                   c
                                   missing-numerics
                                   default-numeric-value
                                   eps
                                   field-codings
                                   normalize
                                   regularization
                                   seed
                                   delete-resources)
  (let (options (get_logistic_options balance-fields
                                      bias
                                      c
                                      missing-numerics
                                      default-numeric-value
                                      eps
                                      field-codings
                                      normalize
                                      regularization
                                      seed)
        objective-id (if (empty? objective-id)
                         (dataset-get-objective-id dataset-id)
                         objective-id))
    (cross-validation dataset-id k-folds objective-id options {} delete-resources)))


(define cross-validation-output
  (logistic-cross-validation dataset-id
                             k-folds
                             objective-id
                             balance-fields
                             bias
                             c
                             missing-numerics
                             default-numeric-value
                             eps
                             field-codings
                             normalize
                             regularization
                             seed
                             delete-resources))
Description
Inputs
Outputs
License: Creative Commons Zero V1.0
Script FREE
Logistic regression's k-fold cross-validation | BigML.com
Sending Request...
COMPANY

PRODUCT

BUSINESS

TRAINING

GALLERY

License

Embed this Script in your web site

COMPANY

PRODUCT

BUSINESS

TRAINING

GALLERY
