Best-first feature selection with cross-validation

Best-first feature selection with cross-validation | BigML.com
FREE
Details:
Created	Fri, 30 Nov 2018 14:45:00 +0000
Published	Fri, 7 Dec 2018 09:37:46 +0000
Description:
Find the best features for modeling using a greedy algorithm. Extends the best-first feature selection script that only worked with models and used split-evaluation
Tags:
URL:
https://bigml.com/user/pgonzalezcarrizo/gallery/script/5c014cec00a1e56a94001786
Script FREE
Source code
;; This code will eventually be defined as a library.

(define MODEL_OPTIONS ["balance_objective"
                       "input_fields"
                       "missing_splits"
                       "pruning"
                       "weight_field"
                       "objective_weights"
                       "node_threshold"])
(define ENSEMBLE_OPTIONS (concat MODEL_OPTIONS
                                 ["sample_rate"
                                  "replacement"
                                  "randomize"
                                  "number_of_models"
                                  "seed"]))
(define BOOSTED_ENSEMBLE_OPTIONS (concat MODEL_OPTIONS
                                         ["boosting"
                                          "sample_rate"
                                          "replacement"
                                          "randomize"
                                          "seed"]))
(define LOGISTIC_OPTIONS ["balance_fields"
                          "input_fields"
                          "bias"
                          "c"
                          "missing_numerics"
                          "default_numeric_value"
                          "eps"
                          "field_codings"
                          "normalize"
                          "regularization"
                          "seed"])
(define EVALUATION_OPTIONS ["sample_rate"
                            "input_fields"
                            "out_of_bag"
                            "range"
                            "replacement"
                            "ordering"
                            "seed"
                            "missing_strategy"
                            "combiner"])
;; cross-validation
;;
;; creates k-fold cross-validation for a dataset
;; Inputs:
;;   dataset-id: (string) Dataset ID
;;   k-folds: (integer) Number of folds
;;   model-options: (map) Options to use in model/ensemble
;;   evaluation-options: (map) Options to use in evaluation creation
;;   delete-resources: (boolean) Whether to delete all intermediate resources
;;
;; Output: (map) Average and standard deviation of evaluations results
;;
;; Raises:
;;  101: The dataset-id argument is not a string
;;  102: The dataset-id is not a valid dataset ID
;;  103: The k-folds argument is not an integer
;;  104: The k-folds argument is not >= 2
;;  106: The objective field ID is not in the selectable IDs list
;;
(define (cross-validation k-fold-datasets
                          objective-id
                          model-options
                          evaluation-options
                          delete-resources)
  (let (dataset (fetch (k-fold-datasets 0))
        dataset-name (dataset "name" false))
    (check-dataset-objective-id objective-id dataset)
    (let (objective-name (get-objective-name dataset objective-id)
          evaluations (create-k-evaluations k-fold-datasets
                                            objective-name
                                            dataset-name
                                            model-options
                                            evaluation-options
                                            delete-resources)
          ;; commented out till wintermute's new version
          ;; evaluations-average (create-and-wait-evaluation {"evaluations" evaluations})
          average-phi (get-average-phi evaluations)
          standard-deviation  (get-deviation-phi evaluations average-phi)
          phi-stdev (- average-phi standard-deviation))
      (log-info "  model options: " model-options)
      (log-info "    average phi: " average-phi)
      (log-info "    phi-stdev: " phi-stdev)
      (when delete-resources
        (map safe-delete evaluations))
      {"phi" average-phi "stdev" standard-deviation "phi-stdev" phi-stdev})))

;; safe-create
;;
;; Creates a resource catching the error
;;
;; Inputs:
;;   type: (string) Type of resource that we want to create
;;   args: (map) Maps of arguments
;;
;; Output: (resource-id) Created resource id
;; 
(define (safe-create type args)
  (try
    (create type args)
    (catch e
      (create type args))))

;; get-average-phi
;;
;; Obtains average phi score of all evaluations in a list
;;
;; Inputs:
;;   evaluations: (list) List of evaluatins ids
;;
;; Output: (number) Average of all average_phi
;; 
(define (get-average-phi evaluations)
  (/ (iterate (acc 0 ev evaluations)
       (+ acc ((fetch ev) ["result" "model" "average_phi"])))
     (count evaluations)))

;; get-deviation-phi
;;
;; Obtains the standard deviation of all evaluations phi score in a list
;;
;; Inputs:
;;   evaluations: (list) List of evaluatins ids
;;   average-phi: (number) Average of all average_phi
;;
;; Output: (number) Standard deviation of all evaluations
;; 
(define (get-deviation-phi evaluations average-phi)
  (sqrt
    (/ (iterate (acc 0 ev evaluations)
         (+ acc
            (pow 
              ( - ((fetch ev) ["result" "model" "average_phi"]) average-phi)
              2)))
       (count evaluations))))

;; check-resource-id
;;
;; Validates that the argument is a resource ID and its type. Raises an error
;; if otherwise.
;;
;; Inputs:
;;   resource-id: (string) Resource ID
;;   type: (string) Type of resource
;;
;; Output: (string) Checked resource ID
(define (check-resource-id resource-id type)
  (when (not (resource-id? resource-id))
    (raise {"message" (str "Resource ID string expected. Found "
                           resource-id " instead.")
            "code" 101}))
  (when (not (= (resource-type resource-id) type))
    (raise {"message" (str "Failed to find a correct " type " ID.")
            "code" 102}))
  resource-id)


;; check-integer
;;
;; Validates that the argument is an integer. Raises error if otherwise.
;;
;; Inputs:
;;  value: (number) Integer to be checked
;;  minimum: (number) Minimum value (false if not set)
;;  maximum: (number) Maximum value (false if not set)
;;
;; Output: (number) Checked integer
(define (check-integer value minimum maximum)
  (when (not (integer? value))
    (raise {"message" (str "Integer value expected. Found " value " instead.")
            "code" 103}))
  (when (and minimum (< value minimum))
    (raise {"message" (str "Minimum accepted value is " minimum ". " value
                           " found.")
            "code" 104}))
  (when (and maximum (> value maximum))
    (raise {"message" (str "Maximum accepted value is " maximum ". " value
                           " found.")
            "code" 105}))
  value)

;; choosable-objective-ids
;;
;; List of IDs of the fields in the dataset that can be chosen as objective
;; field.
;;
;; Inputs:
;;  fields: (map) Fields structure
;; Output: (list) list of field IDs
(define (choosable-objective-ids fields)
  (let (field-val (lambda (fid k) (fields [fid k] false))
        objective-types ["categorical", "numeric"]
        pref? (lambda (k) (field-val k "preferred"))
        pred? (lambda (k) (member? (field-val k "optype") objective-types)))
    (filter (lambda (x) (and (pref? x) (pred? x))) (keys fields))))


;; check-dataset-objective-id
;;
;; Validates that the argument is a valid objective id in the reference
;; dataset.
;;
;; Inputs:
;;  objective-id: (string) ID of the objective field
;;  dataset: (map) Dataset resource information
;;
;; Output: (string) Checked objective field ID
(define (check-dataset-objective-id objective-id dataset)
  (let (fields (dataset "fields" {})
        objective-ids (choosable-objective-ids fields))
    (when (not (member? objective-id objective-ids))
      (raise {"message" (str "Failed to find the objective ID in the dataset"
                             " choosable fields.")
              "code" 106}))))

;; get-objective-name
;;
;; Returns the name of the field used as objective field
;;
;; Inputs:
;;  dataset: (map) Dataset resource info
;;  objective-id: (string) ID of the objective field
;;
;; Outputs: (string) Name of the objective field

(define (get-objective-name dataset objective-id)
  (dataset ["fields" objective-id "name"] false))


;; create-k-folds
;;
;; creating k-fold splits from a dataset
;;
;; Inputs:
;;   dataset-id: (string) Dataset ID
;;   k-folds: (integer) Number of folds
;;
;; Output: (list) List of dataset IDs
;;
(define (create-k-folds dataset-id k-folds)
  (let (k-fold-fn (lambda (x)
                    (safe-create "dataset"
                                 {"origin_dataset" dataset-id
                                  "row_offset" x
                                  "row_step" k-folds
                                  "new_fields" [{"name" "k_fold"
                                                 "field" (str x)}]}))
        dataset-ids (map k-fold-fn (range 0 k-folds)))
    (wait* dataset-ids)))

;; pair-k-folds
;;
;; Builds a list of pairs of hold-out and complementary datasets for all
;; the k-fold dataset IDs.
;;
;; Inputs:
;;   dataset-ids: (list) List of the k-fold dataset IDs
;;
;; Output: (list) List of pairs [hold-out dataset, multidataset with the rest]
;;
(define (pair-k-folds dataset-ids)
  (map (lambda(x) [(nth dataset-ids x) (concat (take x dataset-ids)
                                               (drop (+ x 1) dataset-ids))])
       (range 0 (count dataset-ids))))


;; create-k-models
;;
;; Creates the models for a set of k-fold datasets
;;
;; Inputs:
;;   type: (string) type of model (model or ensemble)
;;   multidatasets: (list) List of lists of datset IDs once a k-fold is
;;                         excluded
;;   objective-name: (string) name of the objective field
;;   model-options: (map) Options for the model or ensemble
;;
;; Output: (list) model IDs
;;
(define (create-k-models type multidatasets objective-name model-options)
  (wait* (for (multidataset multidatasets)
            (safe-create type
                         (merge {"datasets" multidataset
                                 "objective_field" objective-name}
                                model-options)))))


;; create-k-evaluations
;;
;; Creates the models/ensembles and evaluations for a set of k-fold datasets
;;
;; Inputs:
;;   dataset-ids: (list) List of the k-fold dataset IDs
;;   objective-name: (string) Objective field name
;;   dataset-name: (string) Name of the origin dataset
;;   model-options: (map) Options used to build the models/ensembles
;;   evaluation-options: (map) Options used to build evaluations
;;   delete-resources: (boolean) Whether to delete all intermediate resources
;;
;; Output: (list) List of evaluation IDs
;;
(define (create-k-evaluations dataset-ids
                              objective-name
                              dataset-name
                              model-options
                              evaluation-options
                              delete-resources)
  (let (number-of-models (model-options "number_of_models" 1)
        regularization (model-options "regularization" false)
        boosting? (not (empty? (model-options "boosting" {})))
        k-fold-pairs (pair-k-folds dataset-ids)
        ensemble-options (select-keys model-options ENSEMBLE_OPTIONS)
        boosted-ensemble-options (select-keys model-options
                                              BOOSTED_ENSEMBLE_OPTIONS)
        model-options (select-keys model-options MODEL_OPTIONS)
        logistic-options (select-keys model-options LOGISTIC_OPTIONS)
        evaluation-options (select-keys evaluation-options
                                        EVALUATION_OPTIONS)
        type (if (or (> number-of-models 1)
                     boosting?) "ensemble" "model")
        multidatasets (map last k-fold-pairs)
        models (cond (> number-of-models 1)
                     (create-k-models type
                                      multidatasets
                                      objective-name
                                      ensemble-options)
                     regularization
                     (create-k-models type
                                      multidatasets
                                      objective-name
                                      logistic-options)
                     boosting?
                     (create-k-models type
                                      multidatasets
                                      objective-name
                                      boosted-ensemble-options)
                     (create-k-models type
                                      multidatasets
                                      objective-name
                                      model-options))
        evaluations (iterate (es []
                              id dataset-ids
                              mid models
                              idx (range 1 (+ 1 (count dataset-ids))))
                      (let (name (str "Evaluation tested with subset "
                                      idx
                                      " of " dataset-name)
                            opts (assoc evaluation-options "name" name))
                       (append es (safe-create "evaluation"
                                               (merge {"dataset" id
                                                       "model" mid}
                                                      opts))))))
    (wait* evaluations)
    (when delete-resources
      (map safe-delete models))
    evaluations))

;; Script

;;get-model-options
;;
;; maps the options to be used in models
;; Inputs:
;;   missing-splits: (boolen) Sets the missing_splits flag
;;   stat-pruning: (boolean) Sets the statistical pruning flag
;;   balance-objective: (boolean) Sets the balance_objective flag
;;   weight-field: (string) ID of the field to be used as weight (weight_field)
;;   objetive-weights: (list) List of values to be used as objective_weights
;;   node-threshold: (integer) Maximum number of nodes in the model
;; Output: (map) options map
;;
(define (get-model-options missing-splits
                           stat-pruning
                           balance-objective
                           weight-field
                           objective-weights
                           node-threshold)
  (let (options {}
    options (assoc options "missing_splits" missing-splits)
    options (assoc options "stat_pruning" stat-pruning)
    options (assoc options "balance_objective" balance-objective)
    options (if (not (empty? weight-field))
                (assoc options "weight_field" weight-field)
                options)
    options (if (not (empty? objective-weights))
                (assoc options "objective_weights" objective-weights)
                options)
    options (if (not (= node-threshold -1))
                (assoc options "node_threshold" node-threshold)
                options))
    options))

;;get-ensemble-options
;;
;; maps the options to be used in esembles
;; Inputs:
;;   number-of-models: (integer) Number of models in the ensemble
;;   missing-splits: (boolean) Sets the missing_splits flag
;;   stat-pruning: (boolean) Sets the pruning flag
;;   balance-objective: (boolean) Sets the balance_objective flag
;;   weight-field: (string) ID of the field to be used as weight (weight_field)
;;   objective-weights: (list) List of values to be used as objective_weights
;;   node-threshold: (integer) Maximum number of nodes in the model
;;   sample-rate: (float) Percentage of instances used as sample
;;   replacement: (boolean) Sets the replacement flag
;;   randomize: (boolean) Sets the randomize flag
;;   seed: (string) Seed used in random samplings
;; Output: (map) options map
;;
(define (get-ensemble-options number-of-models
                              missing-splits
                              stat-pruning
                              balance-objective
                              weight-field
                              objective-weights
                              node-threshold
                              sample-rate
                              replacement
                              randomize
                              seed)
  (let (options (get-model-options missing-splits
                                   stat-pruning
                                   balance-objective
                                   weight-field
                                   objective-weights
                                   node-threshold)
        options (if (> number-of-models 1)
                    (assoc options "number_of_models" number-of-models)
                    (assoc options "number_of_models" 10))
        options (merge options {"sample_rate" sample-rate
                                "replacement" replacement
                                "randomize" randomize}))
    (if (empty? seed)
        options
        (assoc options "seed" seed))))

;; safe-delete
;;
;; deletes resources ignoring errors
;; Inputs:
;;   id: (resource-id) resource to be deleted
;;
;; Output: (boolean) true if successful, false if not
;;
(define (safe-delete id)
  (try (delete id)
       (catch e
         (log-info (str "Error deleting resource " id " ignored"))
         false)))

;-----------------------------------------------------------------


;; select-features
;;
;; Do best-first feature selection.  Given a dataset and a target
;; number of features iteratively construct models for each feature,
;; evaluate them, and add the feature corresponding to the best
;; cv to the running set of features.  Stop when you reach the
;; target number, you run out of features, or if you don't have enough
;; performance improvement during some iterations
;;
;; Inputs:
;;  dataset-id: (dataset-id) Dataset ID
;;  nfeatures: (number) Number of features that we want to obtain
;;  objective-id: (resource-id) ID of the objective field
;;  options: (map) Options to use in model/ensemble
;;  k-folds: (number) Number of folds
;;  pre-selected-fields: (list) List of IDs of pre-selected fields
;;  early-stop-performance: (number) (%) Minimum required performance improvement
;;  max-low-perf-iterations: (number) (%) Maximum number of iterations with bad
;;                                    performance, as percentage of the number
;;                                    of initial features
;;
;; Output: (map) Response with selected fields names and iterations info
;;
(define (select-features dataset-id
                         nfeatures
                         objective-id
                         options
                         k-folds
                         pre-selected-fields
                         early-stop-performance
                         max-low-perf-iterations)
  (let (obj-id (get-objective dataset-id objective-id)
	k-fold-datasets (create-k-folds dataset-id k-folds)
        fields (resource-fields dataset-id)
	max-bad (* (/ max-low-perf-iterations 100) (count fields))
	max-bad (max 1 max-bad)
        pre-selected-fields (map (lambda (x) ((find-field fields x) "id"))
                          pre-selected-fields)
        _ (log-info pre-selected-fields)
        input-ids (default-inputs dataset-id obj-id)
        important-ids (important-fields dataset-id obj-id)
        potentials (make-complementary important-ids pre-selected-fields))
    (loop (selected pre-selected-fields
           selected-names (feature-names dataset-id pre-selected-fields)
           potentials potentials
           queue []
           bad-count 0)

      (if (or (= bad-count max-bad) 
              (>= (count selected) nfeatures) 
              (empty? potentials))
        (response dataset-id k-fold-datasets selected queue bad-count)
        (let (_ (log-info "Making new candidates...") 
              candidates (make-candidates selected
                                          (map (lambda (x) (x "feature"))
                                          potentials))
              _ (log-info "Candidates..." candidates)
              next-feat-info (select-feature k-fold-datasets
                                             obj-id
                                             options
                                             candidates
                                             potentials)
              bad-count (check-bad-performance queue 
                                               next-feat-info 
                                               early-stop-performance 
                                               bad-count)
              _ (log-info "Bad count value is..." bad-count)
              selected (cons (next-feat-info "feature") selected)
              selected-names (feature-names dataset-id selected)
              queue (cons {"features" selected-names
                            "phi" (next-feat-info "phi")
                            "stdev" (next-feat-info "stdev")
                            "phi-stdev" (next-feat-info "phi-stdev")} 
                          queue))
	  (if (= bad-count max-bad)
              (log-info "Finished due to bad performance on " 
                        max-bad 
                        " iterations")
              (log-info "Selected feature: " 
                        (next-feat-info "feature")))
          (recur selected
                 selected-names
                 (make-complementary potentials [(next-feat-info "feature")])
                 queue
                 bad-count))))))

;; check-bad-performance
;;
;; Checks if the performance improvement of the  last feature (with respect to 
;; the last feature with good performance) is higher than 
;; the threshold value. We understand 'performance' as the evaluation's
;; average phi minus its standard deviation
;;
;; Inputs:
;;   queue: (list) List of maps with results from previous iterations
;;   next-feat-info: (map) Results from last selected feature
;;   early-stop-performance: (number) Minimum required performance improvement
;;   bad-count: (number) Current number of features with bad performance
;;
;; Output: (number) Updated number of features with bad performance
;;
(define (check-bad-performance queue 
                               next-feat-info 
                               early-stop-performance 
                               bad-count)
 (if (< (count queue) 1) 
     bad-count 
     (let  (last-est-phi-stdev (next-feat-info "phi-stdev")
            ;; Last feature with good performance
            previous-est-phi-stdev ((queue bad-count) "phi-stdev")
            performance (- (* (/ last-est-phi-stdev previous-est-phi-stdev) 100) 
                           100))
       (log-info "Performance improvement of last feature: " performance)
       (if (< performance early-stop-performance)
           (+ bad-count 1)
           0))))

;; A simple function to get the max value in a list
(define (get-max xs) (reduce (lambda (x y) (if (> x y) x y)) (head xs) xs))


;; Filters the features in one list to be complementary with the ones in
;; another list
(define (make-complementary y-list x-list)
  (iterate (acc y-list x x-list)
    (filter (lambda (y) (not (= x (y "feature")))) acc)))

;; response
;;
;; Clean the created datasets, removes last features with bad performance
;; and deliver a human-readable response
;;
;; Inputs:
;;    dataset-id: (dataset-id) Dataset id
;;    k-fold-datasets: (list) List of dataset ids
;;    selected: (list) List of selected fields ids
;;    queue: (list) List of maps with results from previous iterations
;;    bad-count: (number) Current number of features with bad performance
;;   
;;
;; Output: (map) Response with selected fields names and iterations info
;;
(define (response dataset-id k-fold-datasets selected queue bad-count)
 ;; We have to remove fields with bad performance 
 (let  (selected (if (> bad-count 0)
                     (reverse (drop  bad-count selected))
                     (reverse selected)))
   ;; Clean the created datasets
   (map safe-delete k-fold-datasets)
   {"selected-fields" (feature-names dataset-id selected)
    "iterations-info" queue}))


;; Get feature names given ids
(define (feature-names dataset-id ids)
  (let (fields ((fetch dataset-id) "fields"))
    (map (lambda (id) (fields [id "name"])) ids)))



;; Get the default set of input fields for this dataset (all preferred
;; fields minus the objective field).
(define (default-inputs dataset-id obj-id)
  (let (fields ((fetch dataset-id) "fields")
        fids (keys fields))
    (filter (lambda (k) (and (fields [k "preferred"] true) (not (= obj-id k))))
            fids)))


;; Get the set of input fields for this dataset by using the importance of the
;; fields in a random decision forest
(define (important-fields dataset-id obj-id)
  (let (ensemble-id (create-and-wait-ensemble dataset-id
                                              {"randomize" true
                                               "objective_field" obj-id})
        ensemble (fetch ensemble-id)
        importance (ensemble "importance")
        importance (for (key (keys importance))
                     {"feature" key "phi" (importance key)}))
    (reverse (sort-by-key "phi" importance))))


;; Make a list of input fields candidates given the potential
;; features in potentials.
(define (make-candidates selected potentials)
  (let (make-req (lambda (fid)
                   (cons fid selected)))
    (map make-req potentials)))


;; select-feature
;;
;; Given a set of dataset folds, a list of potential features, and a list of
;; model ids corresponding to those features, select the best
;; potential feature by performing an evaluation on each model and
;; returning the feature with the best performance.
;;
;; Inputs:
;;    k-fold-datasets: (list) List of Dataset IDs
;;    objective-id: (resource-id) Objective field ID
;;    options: (map) Options to use in model/ensemble
;;    candidates: (list) List of the candidates fields combinations
;;                that we want to test
;;    potentials: (potentials) Potential fields IDs to be selected
;;   
;;
;; Output: (map) Response with selected field info
;;
(define (select-feature k-fold-datasets
                        objective-id
                        options
                        candidates
                        potentials)
  (let (make-attrs (lambda (inp) (assoc options "input_fields" inp))
        potentials (map (lambda (x) (x "feature")) potentials)
        all-reqs (map make-attrs candidates)
        cvs (map (lambda (e-attrs) (cross-validation k-fold-datasets
                                                     objective-id
                                                     e-attrs
                                                     {}
                                                     true)) all-reqs)
        ;;vs (map (lambda (ev) (ev ["result" "model" "average_phi"] 0)) cvs)
        ;; Obtains phi-stdev from each cross validation
        vs (map (lambda (ev) (ev ["phi-stdev"])) cvs)
        value-map (make-map potentials vs)
        max-val (get-max vs)
        choose-best (lambda (id) (and (= max-val (value-map id)) id))
        phi (some (lambda (n) (if (= (n ["phi-stdev"]) max-val)
                                  (n ["phi"]) 
                                  false)) 
             cvs)
        stdev (some (lambda (n) (when (= max-val (n "phi-stdev")) (n "stdev"))) 
               cvs))
    {"feature" (some choose-best potentials)
     "phi" phi
     "stdev" stdev
     "phi-stdev" max-val}))

;; get-objective 
;;
;; Retrieves and checks whether the objective field of the given
;; dataset is categorical, raising an error otherwise.
;;
;; Inputs:
;;    ds-id: (dataset-id) Dataset ID
;;    obj-id: (resource-id) ID of objective field
;;   
;;
;; Output: (resource-id): ID of objective field
;;
(define (get-objective ds-id obj-id)
  (let (obj-id (if (empty? obj-id)
                   (dataset-get-objective-id ds-id)
                   obj-id)
        fields (resource-fields ds-id)
        obj-id ((find-field fields obj-id) "id")
        otype (fields [obj-id "optype"] "missing"))
    (when (not (= "categorical" otype))
      (raise (str "The dataset's objective field must be categorical, "
                  "but is " otype)))
    obj-id))



;; output-features
;;
;; Result from best-first feature selection algorithm
;; Output: (map) Response with selected fields names and iterations info
;;
(define output-features
  (select-features dataset-id 
                   max-n 
                   objective-id 
                   options 
                   k-folds 
                   pre-selected-fields
                   early-stop-performance
                   max-low-perf-iterations))

;; output-dataset
;;
;; Creates new dataset with only the selected fields from the execution of
;; the best-first feature selection algorithm
;; Output: (dataset) Dataset with the selected fields and the objective id
(define output-dataset
  (let (dataset (fetch dataset-id)
        obj-id (get-objective dataset-id objective-id)
        obj-name  (get-objective-name dataset obj-id) )
    (create-and-wait-dataset {"origin_dataset" dataset-id
                              "input_fields"
                              (append (output-features "selected-fields") 
                                      obj-name)
                              "name"
                              (str (dataset "name")
                                   "[BestFeatures]")})))
Description
Inputs
Outputs
License: GNU V3.0
Script FREE
Best-first feature selection with cross-validation | BigML.com
Sending Request...
COMPANY

PRODUCT

BUSINESS

TRAINING

GALLERY

License

Embed this Script in your web site

COMPANY

PRODUCT

BUSINESS

TRAINING

GALLERY
