## sudo vim /etc/heketi/heketi.json
{
  "_port_comment": "Heketi Server Port Number",
  "port": "8080",

        "_enable_tls_comment": "Enable TLS in Heketi Server",
        "enable_tls": false,

        "_cert_file_comment": "Path to a valid certificate file",
        "cert_file": "",

        "_key_file_comment": "Path to a valid private key file",
        "key_file": "",


  "_use_auth": "Enable JWT authorization. Please enable for deployment",
  "use_auth": false,

  "_jwt": "Private keys for access",
  "jwt": {
    "_admin": "Admin has access to all APIs",
    "admin": {
      "_key_comment": "Set the admin key in the next line",
      "key": "############" 
    },
    "_user": "User only has access to /volumes endpoint",
    "user": {
      "_key_comment": "Set the user key in the next line",
      "key": "################"
    }
  },

  "_backup_db_to_kube_secret": "Backup the heketi database to a Kubernetes secret when running in Kubernetes. Default is off.",
  "backup_db_to_kube_secret": false,

  "_profiling": "Enable go/pprof profiling on the /debug/pprof endpoints.",
  "profiling": false,

  "_glusterfs_comment": "GlusterFS Configuration",
  "glusterfs": {
    "_executor_comment": [
      "Execute plugin. Possible choices: mock, ssh",
      "mock: This setting is used for testing and development.",
      "      It will not send commands to any node.",
      "ssh:  This setting will notify Heketi to ssh to the nodes.",
      "      It will need the values in sshexec to be configured.",
      "kubernetes: Communicate with GlusterFS containers over",
      "            Kubernetes exec api."
    ],
    "executor": "ssh",

    "_sshexec_comment": "SSH username and private key file information",
    "sshexec": {
      "keyfile": "/etc/heketi/heketi_key",
      "user": "root",
      "port": "22",
      "fstab": "/etc/fstab"
    },

    "_db_comment": "Database file name",
    "db": "/var/lib/heketi/heketi.db",

     "_refresh_time_monitor_gluster_nodes": "Refresh time in seconds to monitor Gluster nodes",
    "refresh_time_monitor_gluster_nodes": 120,

    "_start_time_monitor_gluster_nodes": "Start time in seconds to monitor Gluster nodes when the heketi comes up",
    "start_time_monitor_gluster_nodes": 10,

    "_loglevel_comment": [
      "Set log level. Choices are:",
      "  none, critical, error, warning, info, debug",
      "Default is warning"
    ],
    "loglevel" : "debug",

    "_auto_create_block_hosting_volume": "Creates Block Hosting volumes automatically if not found or exsisting volume exhausted",
    "auto_create_block_hosting_volume": true,

    "_block_hosting_volume_size": "New block hosting volume will be created in size mentioned, This is considered only if auto-create is enabled.",
    "block_hosting_volume_size": 500,

    "_block_hosting_volume_options": "New block hosting volume will be created with the following set of options. Removing the group gluster-block option is NOT recommended. Additional options can be added next to it separated by a comma.",
    "block_hosting_volume_options": "group gluster-block",

    "_pre_request_volume_options": "Volume options that will be applied for all volumes created. Can be overridden by volume options in volume create request.",
    "pre_request_volume_options": "",

    "_post_request_volume_options": "Volume options that will be applied for all volumes created. To be used to override volume options in volume create request.",
    "post_request_volume_options": ""
  }
}

 "_jwt": "Private keys for access",  "jwt": {
    "_admin": "Admin has access to all APIs",
    "admin": {
      "_key_comment": "Set the admin key in the next line",
      "key": "###############" ## secret for user with name of  admin
    },
    "_user": "User only has access to /volumes endpoint",
    "user": {
      "_key_comment": "Set the user key in the next line",
      "key": "##############" ## secret for user with name of  user
    }
  },

sudo ssh-keygen -f /etc/heketi/heketi_key -t rsa -N ' '
sudo chown -R heketi:heketi /etc/heketi/

sudo vim /etc/systemd/system/heketi.service[Unit]
Description=Heketi Server

[Service]
Type=simple
WorkingDirectory=/var/lib/heketi
EnvironmentFile=-/etc/heketi/heketi.env
User=heketi
ExecStart=/usr/local/bin/heketi --config=/etc/heketi/heketi.json
Restart=on-failure
StandardOutput=syslog
StandardError=syslog

[Install]
WantedBy=multi-user.target

# vim secret.yml
apiVersion: v1
kind: Secret
metadata:
  name: heketi-secret
  namespace: default
type: "kubernetes.io/glusterfs"
data:
  key: ######################

echo -n "PASSWORD" | base64

kubectl create -f secret.yml 
OR 
kubectl apply -f secret.yml

# vim StorageClass.yml
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
  name: gluster
provisioner: kubernetes.io/glusterfs
reclaimPolicy: Retain
volumeBindingMode: Immediate 
allowVolumeExpansion: true
parameters:
  resturl: "http://gluster-gw-heketi:8080"
  clusterid: "#########################"
  restauthenabled: "true"
  restuser: "admin"
#  restuserkey: "###########"
  secretNamespace: "default"
  secretName: "heketi-secret"
  volumetype: "replicate:3"
  volumenameprefix: "k8s-dynamic-provisioning"

heketi-cli cluster list
Clusters:
Id:################################ [file][block]

#vim pvc.yml
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: gluster1
spec:
  accessModes:
  - ReadWriteMany
  resources:
    requests:
      storage: 1Gi
  storageClassName: gluster