automatic import of glusterfsopeneuler24.03_LTS

1 files changed, 1077 insertions, 0 deletions

diff --git a/0153-features-cloudsync-Added-some-new-functions.patch b/0153-features-cloudsync-Added-some-new-functions.patch
new file mode 100644
index 0000000..d28a7a5
--- /dev/null
+++ b/0153-features-cloudsync-Added-some-new-functions.patch
@@ -0,0 +1,1077 @@
+From 90254e4ae9455fa0a126f83700978a9314eb79ea Mon Sep 17 00:00:00 2001
+From: Anuradha Talur <atalur@commvault.com>
+Date: Thu, 29 Nov 2018 12:54:21 -0800
+Subject: [PATCH 153/169] features/cloudsync : Added some new functions
+
+This patch contains the following changes:
+1) Store ID info will now be stored in the inode ctx
+2) Added new readv type where read is made directly
+   from the remote store. This choice is made by
+   volume set operation.
+3) cs_forget() was missing. Added it.
+
+backport of:https://review.gluster.org/#/c/glusterfs/+/21757/
+
+> Change-Id: Ie3232b3d7ffb5313a03f011b0553b19793eedfa2
+> fixes: bz#1642168
+> Signed-off-by: Anuradha Talur <atalur@commvault.com>
+
+Change-Id: I089e5a8c93049cf6bfabf011673796e38e78d7ee
+Signed-off-by: Susant Palai <spalai@redhat.com>
+Reviewed-on: https://code.engineering.redhat.com/gerrit/172192
+Tested-by: RHGS Build Bot <nigelb@redhat.com>
+Reviewed-by: Atin Mukherjee <amukherj@redhat.com>
+---
+ xlators/features/cloudsync/src/cloudsync-common.c  |  16 +
+ xlators/features/cloudsync/src/cloudsync-common.h  |  35 ++
+ xlators/features/cloudsync/src/cloudsync-fops-c.py |  12 +-
+ .../features/cloudsync/src/cloudsync-mem-types.h   |   1 +
+ xlators/features/cloudsync/src/cloudsync.c         | 600 ++++++++++++++++++---
+ xlators/features/cloudsync/src/cloudsync.h         |  20 +
+ xlators/mgmt/glusterd/src/glusterd-volume-set.c    |   7 +-
+ 7 files changed, 597 insertions(+), 94 deletions(-)
+
+diff --git a/xlators/features/cloudsync/src/cloudsync-common.c b/xlators/features/cloudsync/src/cloudsync-common.c
+index aee1f06..445a31b 100644
+--- a/xlators/features/cloudsync/src/cloudsync-common.c
++++ b/xlators/features/cloudsync/src/cloudsync-common.c
+@@ -11,6 +11,20 @@
+ #include "cloudsync-common.h"
+ 
+ void
++cs_xattrinfo_wipe(cs_local_t *local)
++{
++    if (local->xattrinfo.lxattr) {
++        if (local->xattrinfo.lxattr->file_path)
++            GF_FREE(local->xattrinfo.lxattr->file_path);
++
++        if (local->xattrinfo.lxattr->volname)
++            GF_FREE(local->xattrinfo.lxattr->volname);
++
++        GF_FREE(local->xattrinfo.lxattr);
++    }
++}
++
++void
+ cs_local_wipe(xlator_t *this, cs_local_t *local)
+ {
+     if (!local)
+@@ -40,5 +54,7 @@ cs_local_wipe(xlator_t *this, cs_local_t *local)
+     if (local->remotepath)
+         GF_FREE(local->remotepath);
+ 
++    cs_xattrinfo_wipe(local);
++
+     mem_put(local);
+ }
+diff --git a/xlators/features/cloudsync/src/cloudsync-common.h b/xlators/features/cloudsync/src/cloudsync-common.h
+index 7b3520c..11d2334 100644
+--- a/xlators/features/cloudsync/src/cloudsync-common.h
++++ b/xlators/features/cloudsync/src/cloudsync-common.h
+@@ -14,9 +14,23 @@
+ #include <glusterfs/call-stub.h>
+ #include <glusterfs/xlator.h>
+ #include <glusterfs/syncop.h>
++#include <glusterfs/compat-errno.h>
+ #include "cloudsync-mem-types.h"
+ #include "cloudsync-messages.h"
+ 
++typedef struct cs_loc_xattr {
++    char *file_path;
++    uuid_t uuid;
++    uuid_t gfid;
++    char *volname;
++} cs_loc_xattr_t;
++
++typedef struct cs_size_xattr {
++    uint64_t size;
++    uint64_t blksize;
++    uint64_t blocks;
++} cs_size_xattr_t;
++
+ typedef struct cs_local {
+     loc_t loc;
+     fd_t *fd;
+@@ -34,10 +48,25 @@ typedef struct cs_local {
+     int call_cnt;
+     inode_t *inode;
+     char *remotepath;
++
++    struct {
++        /* offset, flags and size are the information needed
++         * by read fop for remote read operation. These will be
++         * populated in cloudsync read fop, before being passed
++         * on to the plugin performing remote read.
++         */
++        off_t offset;
++        uint32_t flags;
++        size_t size;
++        cs_loc_xattr_t *lxattr;
++    } xattrinfo;
++
+ } cs_local_t;
+ 
+ typedef int (*fop_download_t)(call_frame_t *frame, void *config);
+ 
++typedef int (*fop_remote_read_t)(call_frame_t *, void *);
++
+ typedef void *(*store_init)(xlator_t *this);
+ 
+ typedef int (*store_reconfigure)(xlator_t *this, dict_t *options);
+@@ -48,6 +77,7 @@ struct cs_remote_stores {
+     char *name;                    /* store name */
+     void *config;                  /* store related information */
+     fop_download_t dlfop;          /* store specific download function */
++    fop_remote_read_t rdfop;       /* store specific read function */
+     store_init init;               /* store init to initialize store config */
+     store_reconfigure reconfigure; /* reconfigure store config */
+     store_fini fini;
+@@ -59,11 +89,15 @@ typedef struct cs_private {
+     struct cs_remote_stores *stores;
+     gf_boolean_t abortdl;
+     pthread_spinlock_t lock;
++    gf_boolean_t remote_read;
+ } cs_private_t;
+ 
+ void
+ cs_local_wipe(xlator_t *this, cs_local_t *local);
+ 
++void
++cs_xattrinfo_wipe(cs_local_t *local);
++
+ #define CS_STACK_UNWIND(fop, frame, params...)                                 \
+     do {                                                                       \
+         cs_local_t *__local = NULL;                                            \
+@@ -90,6 +124,7 @@ cs_local_wipe(xlator_t *this, cs_local_t *local);
+ 
+ typedef struct store_methods {
+     int (*fop_download)(call_frame_t *frame, void *config);
++    int (*fop_remote_read)(call_frame_t *, void *);
+     /* return type should be the store config */
+     void *(*fop_init)(xlator_t *this);
+     int (*fop_reconfigure)(xlator_t *this, dict_t *options);
+diff --git a/xlators/features/cloudsync/src/cloudsync-fops-c.py b/xlators/features/cloudsync/src/cloudsync-fops-c.py
+index 3122bd3..a7a2201 100755
+--- a/xlators/features/cloudsync/src/cloudsync-fops-c.py
++++ b/xlators/features/cloudsync/src/cloudsync-fops-c.py
+@@ -137,15 +137,15 @@ cs_@NAME@_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                         } else {
+                                 __cs_inode_ctx_update (this, fd->inode, val);
+                                 gf_msg (this->name, GF_LOG_INFO, 0, 0,
+-                                        " state = %ld", val);
++                                        " state = %" PRIu64, val);
+ 
+                                 if (local->call_cnt == 1 &&
+                                     (val == GF_CS_REMOTE ||
+                                      val == GF_CS_DOWNLOADING))  {
+                                         gf_msg (this->name, GF_LOG_INFO, 0,
+                                                 0, " will repair and download "
+-                                                "the file, current state : %ld",
+-                                                val);
++                                                "the file, current state : %"
++                                                PRIu64, val);
+                                         goto repair;
+                                 } else {
+                                         gf_msg (this->name, GF_LOG_ERROR, 0, 0,
+@@ -274,7 +274,7 @@ fd_ops = ['readv', 'writev', 'flush', 'fsync', 'fsyncdir', 'ftruncate',
+ # These are the current actual lists used to generate the code
+ 
+ # The following list contains fops which are fd based that modifies data
+-fd_data_modify_op_fop_template = ['readv', 'writev', 'flush', 'fsync',
++fd_data_modify_op_fop_template = ['writev', 'flush', 'fsync',
+                                   'ftruncate', 'rchecksum', 'fallocate',
+                                   'discard', 'zerofill', 'seek']
+ 
+@@ -284,8 +284,8 @@ loc_stat_op_fop_template = ['lookup', 'stat', 'discover', 'access', 'setattr',
+                             'getattr']
+ 
+ # These fops need a separate implementation
+-special_fops = ['readdirp', 'statfs', 'setxattr', 'unlink', 'getxattr',
+-                'truncate', 'fstat']
++special_fops = ['statfs', 'setxattr', 'unlink', 'getxattr',
++                'truncate', 'fstat', 'readv']
+ 
+ def gen_defaults():
+     for name in ops:
+diff --git a/xlators/features/cloudsync/src/cloudsync-mem-types.h b/xlators/features/cloudsync/src/cloudsync-mem-types.h
+index 9e6837a..2203464 100644
+--- a/xlators/features/cloudsync/src/cloudsync-mem-types.h
++++ b/xlators/features/cloudsync/src/cloudsync-mem-types.h
+@@ -16,6 +16,7 @@ enum cs_mem_types_ {
+     gf_cs_mt_cs_private_t = gf_common_mt_end + 1,
+     gf_cs_mt_cs_remote_stores_t,
+     gf_cs_mt_cs_inode_ctx_t,
++    gf_cs_mt_cs_lxattr_t,
+     gf_cs_mt_end
+ };
+ #endif /* __CLOUDSYNC_MEM_TYPES_H__ */
+diff --git a/xlators/features/cloudsync/src/cloudsync.c b/xlators/features/cloudsync/src/cloudsync.c
+index fbdcdf7..2240fc3 100644
+--- a/xlators/features/cloudsync/src/cloudsync.c
++++ b/xlators/features/cloudsync/src/cloudsync.c
+@@ -16,6 +16,7 @@
+ #include <glusterfs/call-stub.h>
+ #include "cloudsync-autogen-fops.h"
+ 
++#include <string.h>
+ #include <dlfcn.h>
+ 
+ void
+@@ -72,6 +73,8 @@ cs_init(xlator_t *this)
+ 
+     this->private = priv;
+ 
++    GF_OPTION_INIT("cloudsync-remote-read", priv->remote_read, bool, out);
++
+     /* temp workaround. Should be configurable through glusterd*/
+     per_vol = _gf_true;
+ 
+@@ -135,6 +138,18 @@ cs_init(xlator_t *this)
+ 
+         (void)dlerror();
+ 
++        if (priv->remote_read) {
++            priv->stores->rdfop = store_methods->fop_remote_read;
++            if (!priv->stores->rdfop) {
++                gf_msg(this->name, GF_LOG_ERROR, 0, 0,
++                       "failed to get"
++                       " read fop %s",
++                       dlerror());
++                ret = -1;
++                goto out;
++            }
++        }
++
+         priv->stores->dlfop = store_methods->fop_download;
+         if (!priv->stores->dlfop) {
+             gf_msg(this->name, GF_LOG_ERROR, 0, 0,
+@@ -196,6 +211,22 @@ out:
+     return ret;
+ }
+ 
++int
++cs_forget(xlator_t *this, inode_t *inode)
++{
++    uint64_t ctx_int = 0;
++    cs_inode_ctx_t *ctx = NULL;
++
++    inode_ctx_del(inode, this, &ctx_int);
++    if (!ctx_int)
++        return 0;
++
++    ctx = (cs_inode_ctx_t *)(uintptr_t)ctx_int;
++
++    GF_FREE(ctx);
++    return 0;
++}
++
+ void
+ cs_fini(xlator_t *this)
+ {
+@@ -217,6 +248,9 @@ cs_reconfigure(xlator_t *this, dict_t *options)
+         goto out;
+     }
+ 
++    GF_OPTION_RECONF("cloudsync-remote-read", priv->remote_read, options, bool,
++                     out);
++
+     /* needed only for per volume configuration*/
+     ret = priv->stores->reconfigure(this, options);
+ 
+@@ -242,59 +276,6 @@ out:
+ }
+ 
+ int32_t
+-cs_readdirp_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+-                int32_t op_ret, int32_t op_errno, gf_dirent_t *entries,
+-                dict_t *xdata)
+-{
+-    gf_dirent_t *tmp = NULL;
+-    char *sxattr = NULL;
+-    uint64_t ia_size = 0;
+-    int ret = 0;
+-
+-    list_for_each_entry(tmp, &entries->list, list)
+-    {
+-        ret = dict_get_str(tmp->dict, GF_CS_OBJECT_SIZE, &sxattr);
+-        if (ret) {
+-            gf_msg_trace(this->name, 0, "size xattr found");
+-            continue;
+-        }
+-
+-        ia_size = atoll(sxattr);
+-        tmp->d_stat.ia_size = ia_size;
+-    }
+-
+-    STACK_UNWIND_STRICT(readdirp, frame, op_ret, op_errno, entries, xdata);
+-    return 0;
+-}
+-
+-int32_t
+-cs_readdirp(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+-            off_t off, dict_t *xdata)
+-{
+-    int ret = 0;
+-    int op_errno = ENOMEM;
+-
+-    if (!xdata) {
+-        xdata = dict_new();
+-        if (!xdata) {
+-            goto err;
+-        }
+-    }
+-
+-    ret = dict_set_int32(xdata, GF_CS_OBJECT_SIZE, 1);
+-    if (ret) {
+-        goto err;
+-    }
+-
+-    STACK_WIND(frame, cs_readdirp_cbk, FIRST_CHILD(this),
+-               FIRST_CHILD(this)->fops->readdirp, fd, size, off, xdata);
+-    return 0;
+-err:
+-    STACK_UNWIND_STRICT(readdirp, frame, -1, op_errno, NULL, NULL);
+-    return 0;
+-}
+-
+-int32_t
+ cs_truncate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                 struct iatt *postbuf, dict_t *xdata)
+@@ -305,7 +286,6 @@ cs_truncate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+ 
+     local = frame->local;
+ 
+-    /* Do we need lock here? */
+     local->call_cnt++;
+ 
+     if (op_ret == -1) {
+@@ -320,13 +300,13 @@ cs_truncate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 goto unwind;
+             } else {
+                 __cs_inode_ctx_update(this, local->loc.inode, val);
+-                gf_msg(this->name, GF_LOG_INFO, 0, 0, " state = %ld", val);
++                gf_msg(this->name, GF_LOG_INFO, 0, 0, " state = %" PRIu64, val);
+ 
+                 if (local->call_cnt == 1 &&
+                     (val == GF_CS_REMOTE || val == GF_CS_DOWNLOADING)) {
+                     gf_msg(this->name, GF_LOG_WARNING, 0, 0,
+                            "will repair and download "
+-                           "the file, current state : %ld",
++                           "the file, current state : %" PRIu64,
+                            val);
+                     goto repair;
+                 } else {
+@@ -665,7 +645,7 @@ cs_fstat_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+     if (op_ret == 0) {
+         ret = dict_get_uint64(xdata, GF_CS_OBJECT_STATUS, &val);
+         if (!ret) {
+-            gf_msg_debug(this->name, 0, "state %ld", val);
++            gf_msg_debug(this->name, 0, "state %" PRIu64, val);
+             ret = __cs_inode_ctx_update(this, fd->inode, val);
+             if (ret) {
+                 gf_msg(this->name, GF_LOG_ERROR, 0, 0, "ctx update failed");
+@@ -831,7 +811,7 @@ out:
+     return 0;
+ }
+ 
+-void *
++int
+ cs_download_task(void *arg)
+ {
+     call_frame_t *frame = NULL;
+@@ -842,7 +822,6 @@ cs_download_task(void *arg)
+     fd_t *fd = NULL;
+     cs_local_t *local = NULL;
+     dict_t *dict = NULL;
+-    int *retval = NULL;
+ 
+     frame = (call_frame_t *)arg;
+ 
+@@ -850,13 +829,6 @@ cs_download_task(void *arg)
+ 
+     priv = this->private;
+ 
+-    retval = GF_CALLOC(1, sizeof(int), gf_common_mt_int);
+-    if (!retval) {
+-        gf_msg(this->name, GF_LOG_ERROR, 0, 0, "insufficient memory");
+-        ret = -1;
+-        goto out;
+-    }
+-
+     if (!priv->stores) {
+         gf_msg(this->name, GF_LOG_ERROR, 0, 0,
+                "No remote store "
+@@ -972,20 +944,13 @@ out:
+         local->dlfd = NULL;
+     }
+ 
+-    if (retval) {
+-        *retval = ret;
+-        pthread_exit(retval);
+-    } else {
+-        pthread_exit(&ret);
+-    }
++    return ret;
+ }
+ 
+ int
+ cs_download(call_frame_t *frame)
+ {
+-    int *retval = NULL;
+     int ret = 0;
+-    pthread_t dthread;
+     cs_local_t *local = NULL;
+     xlator_t *this = NULL;
+ 
+@@ -1000,16 +965,406 @@ cs_download(call_frame_t *frame)
+         goto out;
+     }
+ 
+-    ret = gf_thread_create(&dthread, NULL, &cs_download_task, (void *)frame,
+-                           "downloadthread");
++    ret = cs_download_task((void *)frame);
++out:
++    return ret;
++}
+ 
+-    pthread_join(dthread, (void **)&retval);
++int
++cs_set_xattr_req(call_frame_t *frame)
++{
++    cs_local_t *local = NULL;
++    GF_UNUSED int ret = 0;
++
++    local = frame->local;
++
++    /* When remote reads are performed (i.e. reads on remote store),
++     * there needs to be a way to associate a file on gluster volume
++     * with its correspnding file on the remote store. In order to do
++     * that, a unique key can be maintained as an xattr
++     * (GF_CS_XATTR_ARCHIVE_UUID)on the stub file on gluster bricks.
++     * This xattr should be provided to the plugin to
++     * perform the read fop on the correct file. This assumes that the file
++     * hierarchy and name need not be the same on remote store as that of
++     * the gluster volume.
++     */
++    ret = dict_set_str(local->xattr_req, GF_CS_XATTR_ARCHIVE_UUID, "1");
++
++    return 0;
++}
+ 
+-    ret = *retval;
++int
++cs_update_xattrs(call_frame_t *frame, dict_t *xdata)
++{
++    cs_local_t *local = NULL;
++    xlator_t *this = NULL;
++    int size = -1;
++    GF_UNUSED int ret = 0;
++
++    local = frame->local;
++    this = frame->this;
++
++    local->xattrinfo.lxattr = GF_CALLOC(1, sizeof(cs_loc_xattr_t),
++                                        gf_cs_mt_cs_lxattr_t);
++    if (!local->xattrinfo.lxattr) {
++        local->op_ret = -1;
++        local->op_errno = ENOMEM;
++        goto err;
++    }
++
++    gf_uuid_copy(local->xattrinfo.lxattr->gfid, local->loc.gfid);
++
++    if (local->remotepath) {
++        local->xattrinfo.lxattr->file_path = gf_strdup(local->remotepath);
++        if (!local->xattrinfo.lxattr->file_path) {
++            local->op_ret = -1;
++            local->op_errno = ENOMEM;
++            goto err;
++        }
++    }
++
++    ret = dict_get_gfuuid(xdata, GF_CS_XATTR_ARCHIVE_UUID,
++                          &(local->xattrinfo.lxattr->uuid));
++
++    if (ret) {
++        gf_uuid_clear(local->xattrinfo.lxattr->uuid);
++    }
++    size = strlen(this->name) - strlen("-cloudsync") + 1;
++    local->xattrinfo.lxattr->volname = GF_CALLOC(1, size, gf_common_mt_char);
++    if (!local->xattrinfo.lxattr->volname) {
++        local->op_ret = -1;
++        local->op_errno = ENOMEM;
++        goto err;
++    }
++    strncpy(local->xattrinfo.lxattr->volname, this->name, size - 1);
++    local->xattrinfo.lxattr->volname[size - 1] = '\0';
++
++    return 0;
++err:
++    cs_xattrinfo_wipe(local);
++    return -1;
++}
++
++int
++cs_serve_readv(call_frame_t *frame, off_t offset, size_t size, uint32_t flags)
++{
++    xlator_t *this = NULL;
++    cs_private_t *priv = NULL;
++    int ret = -1;
++    fd_t *fd = NULL;
++    cs_local_t *local = NULL;
++
++    local = frame->local;
++    this = frame->this;
++    priv = this->private;
++
++    if (!local->remotepath) {
++        ret = -1;
++        gf_msg(this->name, GF_LOG_ERROR, 0, 0,
++               "remote path not"
++               " available. Check posix logs to resolve");
++        goto out;
++    }
++
++    if (!priv->stores) {
++        gf_msg(this->name, GF_LOG_ERROR, 0, 0,
++               "No remote store "
++               "plugins found");
++        ret = -1;
++        goto out;
++    }
++
++    if (local->fd) {
++        fd = fd_anonymous(local->fd->inode);
++    } else {
++        fd = fd_anonymous(local->loc.inode);
++    }
++
++    local->xattrinfo.size = size;
++    local->xattrinfo.offset = offset;
++    local->xattrinfo.flags = flags;
++
++    if (!fd) {
++        gf_msg("CS", GF_LOG_ERROR, 0, 0, "fd creation failed");
++        ret = -1;
++        goto out;
++    }
++
++    local->dlfd = fd;
++    local->dloffset = offset;
++
++    /*this calling method is for per volume setting */
++    ret = priv->stores->rdfop(frame, priv->stores->config);
++    if (ret) {
++        gf_msg(this->name, GF_LOG_ERROR, 0, 0,
++               "read failed"
++               ", remotepath: %s",
++               local->remotepath);
++        ret = -1;
++        goto out;
++    } else {
++        gf_msg(this->name, GF_LOG_INFO, 0, 0,
++               "read success, path"
++               " : %s",
++               local->remotepath);
++    }
+ 
+ out:
+-    if (retval)
+-        GF_FREE(retval);
++    if (fd) {
++        fd_unref(fd);
++        local->dlfd = NULL;
++    }
++    return ret;
++}
++
++int32_t
++cs_readv_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
++             int32_t op_errno, struct iovec *vector, int32_t count,
++             struct iatt *stbuf, struct iobref *iobref, dict_t *xdata)
++{
++    cs_local_t *local = NULL;
++    int ret = 0;
++    uint64_t val = 0;
++    fd_t *fd = NULL;
++
++    local = frame->local;
++    fd = local->fd;
++
++    local->call_cnt++;
++
++    if (op_ret == -1) {
++        ret = dict_get_uint64(xdata, GF_CS_OBJECT_STATUS, &val);
++        if (ret == 0) {
++            if (val == GF_CS_ERROR) {
++                gf_msg(this->name, GF_LOG_ERROR, 0, 0,
++                       "could not get file state, unwinding");
++                op_ret = -1;
++                op_errno = EIO;
++                goto unwind;
++            } else {
++                __cs_inode_ctx_update(this, fd->inode, val);
++                gf_msg(this->name, GF_LOG_INFO, 0, 0, " state = %" PRIu64, val);
++
++                if (local->call_cnt == 1 &&
++                    (val == GF_CS_REMOTE || val == GF_CS_DOWNLOADING)) {
++                    gf_msg(this->name, GF_LOG_INFO, 0, 0,
++                           " will read from remote : %" PRIu64, val);
++                    goto repair;
++                } else {
++                    gf_msg(this->name, GF_LOG_ERROR, 0, 0,
++                           "second readv, Unwinding");
++                    goto unwind;
++                }
++            }
++        } else {
++            gf_msg(this->name, GF_LOG_ERROR, 0, 0,
++                   "file state "
++                   "could not be figured, unwinding");
++            goto unwind;
++        }
++    } else {
++        /* successful readv => file is local */
++        __cs_inode_ctx_update(this, fd->inode, GF_CS_LOCAL);
++        gf_msg(this->name, GF_LOG_INFO, 0, 0,
++               "state : GF_CS_LOCAL"
++               ", readv successful");
++
++        goto unwind;
++    }
++
++repair:
++    ret = locate_and_execute(frame);
++    if (ret) {
++        goto unwind;
++    }
++
++    return 0;
++
++unwind:
++    CS_STACK_UNWIND(readv, frame, op_ret, op_errno, vector, count, stbuf,
++                    iobref, xdata);
++
++    return 0;
++}
++
++int32_t
++cs_resume_readv(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
++                off_t offset, uint32_t flags, dict_t *xdata)
++{
++    int ret = 0;
++
++    ret = cs_resume_postprocess(this, frame, fd->inode);
++    if (ret) {
++        goto unwind;
++    }
++
++    cs_inodelk_unlock(frame);
++
++    STACK_WIND(frame, cs_readv_cbk, FIRST_CHILD(this),
++               FIRST_CHILD(this)->fops->readv, fd, size, offset, flags, xdata);
++
++    return 0;
++
++unwind:
++    cs_inodelk_unlock(frame);
++
++    cs_common_cbk(frame);
++
++    return 0;
++}
++
++int32_t
++cs_resume_remote_readv(call_frame_t *frame, xlator_t *this, fd_t *fd,
++                       size_t size, off_t offset, uint32_t flags, dict_t *xdata)
++{
++    int ret = 0;
++    cs_local_t *local = NULL;
++    gf_cs_obj_state state = -1;
++    cs_inode_ctx_t *ctx = NULL;
++
++    cs_inodelk_unlock(frame);
++
++    local = frame->local;
++    if (!local) {
++        ret = -1;
++        goto unwind;
++    }
++
++    __cs_inode_ctx_get(this, fd->inode, &ctx);
++
++    state = __cs_get_file_state(this, fd->inode, ctx);
++    if (state == GF_CS_ERROR) {
++        gf_msg(this->name, GF_LOG_ERROR, 0, 0,
++               "status is GF_CS_ERROR."
++               " Aborting readv");
++        local->op_ret = -1;
++        local->op_errno = EREMOTE;
++        ret = -1;
++        goto unwind;
++    }
++
++    /* Serve readv from remote store only if it is remote. */
++    gf_msg_debug(this->name, 0, "status of file %s is %d",
++                 local->remotepath ? local->remotepath : "", state);
++
++    /* We will reach this condition if local inode ctx had REMOTE
++     * state when the control was in cs_readv but after stat
++     * we got an updated state saying that the file is LOCAL.
++     */
++    if (state == GF_CS_LOCAL) {
++        STACK_WIND(frame, cs_readv_cbk, FIRST_CHILD(this),
++                   FIRST_CHILD(this)->fops->readv, fd, size, offset, flags,
++                   xdata);
++    } else if (state == GF_CS_REMOTE) {
++        ret = cs_resume_remote_readv_postprocess(this, frame, fd->inode, offset,
++                                                 size, flags);
++        /* Failed to submit the remote readv fop to plugin */
++        if (ret) {
++            local->op_ret = -1;
++            local->op_errno = EREMOTE;
++            goto unwind;
++        }
++        /* When the file is in any other intermediate state,
++         * we should not perform remote reads.
++         */
++    } else {
++        local->op_ret = -1;
++        local->op_errno = EINVAL;
++        goto unwind;
++    }
++
++    return 0;
++
++unwind:
++    cs_common_cbk(frame);
++
++    return 0;
++}
++
++int32_t
++cs_readv(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
++         off_t offset, uint32_t flags, dict_t *xdata)
++{
++    int op_errno = -1;
++    cs_local_t *local = NULL;
++    int ret = 0;
++    cs_inode_ctx_t *ctx = NULL;
++    gf_cs_obj_state state = -1;
++    cs_private_t *priv = NULL;
++
++    VALIDATE_OR_GOTO(frame, err);
++    VALIDATE_OR_GOTO(this, err);
++    VALIDATE_OR_GOTO(fd, err);
++
++    priv = this->private;
++
++    local = cs_local_init(this, frame, NULL, fd, GF_FOP_READ);
++    if (!local) {
++        gf_msg(this->name, GF_LOG_ERROR, 0, 0, "local init failed");
++        op_errno = ENOMEM;
++        goto err;
++    }
++
++    __cs_inode_ctx_get(this, fd->inode, &ctx);
++
++    if (ctx)
++        state = __cs_get_file_state(this, fd->inode, ctx);
++    else
++        state = GF_CS_LOCAL;
++
++    local->xattr_req = xdata ? dict_ref(xdata) : (xdata = dict_new());
++
++    ret = dict_set_uint32(local->xattr_req, GF_CS_OBJECT_STATUS, 1);
++    if (ret) {
++        gf_msg(this->name, GF_LOG_ERROR, 0, 0,
++               "dict_set failed key:"
++               " %s",
++               GF_CS_OBJECT_STATUS);
++        goto err;
++    }
++
++    if (priv->remote_read) {
++        local->stub = fop_readv_stub(frame, cs_resume_remote_readv, fd, size,
++                                     offset, flags, xdata);
++    } else {
++        local->stub = fop_readv_stub(frame, cs_resume_readv, fd, size, offset,
++                                     flags, xdata);
++    }
++    if (!local->stub) {
++        gf_msg(this->name, GF_LOG_ERROR, 0, 0, "insufficient memory");
++        op_errno = ENOMEM;
++        goto err;
++    }
++
++    if (state == GF_CS_LOCAL) {
++        STACK_WIND(frame, cs_readv_cbk, FIRST_CHILD(this),
++                   FIRST_CHILD(this)->fops->readv, fd, size, offset, flags,
++                   xdata);
++    } else {
++        local->call_cnt++;
++        ret = locate_and_execute(frame);
++        if (ret) {
++            op_errno = ENOMEM;
++            goto err;
++        }
++    }
++
++    return 0;
++
++err:
++    CS_STACK_UNWIND(readv, frame, -1, op_errno, NULL, -1, NULL, NULL, NULL);
++
++    return 0;
++}
++
++int
++cs_resume_remote_readv_postprocess(xlator_t *this, call_frame_t *frame,
++                                   inode_t *inode, off_t offset, size_t size,
++                                   uint32_t flags)
++{
++    int ret = 0;
++
++    ret = cs_serve_readv(frame, offset, size, flags);
+ 
+     return ret;
+ }
+@@ -1059,7 +1414,7 @@ cs_stat_check_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+                 goto err;
+             } else {
+                 ret = __cs_inode_ctx_update(this, inode, val);
+-                gf_msg_debug(this->name, 0, "status : %lu", val);
++                gf_msg_debug(this->name, 0, "status : %" PRIu64, val);
+                 if (ret) {
+                     gf_msg(this->name, GF_LOG_ERROR, 0, 0, "ctx update failed");
+                     local->op_ret = -1;
+@@ -1087,6 +1442,10 @@ cs_stat_check_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+             gf_msg_debug(this->name, 0, "NULL filepath");
+         }
+ 
++        ret = cs_update_xattrs(frame, xdata);
++        if (ret)
++            goto err;
++
+         local->op_ret = 0;
+         local->xattr_rsp = dict_ref(xdata);
+         memcpy(&local->stbuf, stbuf, sizeof(struct iatt));
+@@ -1121,6 +1480,8 @@ cs_do_stat_check(call_frame_t *main_frame)
+         goto err;
+     }
+ 
++    cs_set_xattr_req(main_frame);
++
+     if (local->fd) {
+         STACK_WIND(main_frame, cs_stat_check_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->fstat, local->fd, local->xattr_req);
+@@ -1177,6 +1538,10 @@ cs_common_cbk(call_frame_t *frame)
+                             NULL, NULL, NULL);
+             break;
+ 
++        case GF_FOP_TRUNCATE:
++            CS_STACK_UNWIND(truncate, frame, local->op_ret, local->op_errno,
++                            NULL, NULL, NULL);
++            break;
+         default:
+             break;
+     }
+@@ -1427,7 +1792,7 @@ __cs_inode_ctx_get(xlator_t *this, inode_t *inode, cs_inode_ctx_t **ctx)
+     if (ret)
+         *ctx = NULL;
+     else
+-        *ctx = (cs_inode_ctx_t *)ctxint;
++        *ctx = (cs_inode_ctx_t *)(uintptr_t)ctxint;
+ 
+     return;
+ }
+@@ -1452,7 +1817,7 @@ __cs_inode_ctx_update(xlator_t *this, inode_t *inode, uint64_t val)
+ 
+             ctx->state = val;
+ 
+-            ctxint = (uint64_t)ctx;
++            ctxint = (uint64_t)(uintptr_t)ctx;
+ 
+             ret = __inode_ctx_set(inode, this, &ctxint);
+             if (ret) {
+@@ -1460,7 +1825,7 @@ __cs_inode_ctx_update(xlator_t *this, inode_t *inode, uint64_t val)
+                 goto out;
+             }
+         } else {
+-            ctx = (cs_inode_ctx_t *)ctxint;
++            ctx = (cs_inode_ctx_t *)(uintptr_t)ctxint;
+ 
+             ctx->state = val;
+         }
+@@ -1483,7 +1848,7 @@ cs_inode_ctx_reset(xlator_t *this, inode_t *inode)
+         return 0;
+     }
+ 
+-    ctx = (cs_inode_ctx_t *)ctxint;
++    ctx = (cs_inode_ctx_t *)(uintptr_t)ctxint;
+ 
+     GF_FREE(ctx);
+     return 0;
+@@ -1532,6 +1897,57 @@ cs_resume_postprocess(xlator_t *this, call_frame_t *frame, inode_t *inode)
+ out:
+     return ret;
+ }
++
++int32_t
++__cs_get_dict_str(char **str, dict_t *xattr, const char *name, int *errnum)
++{
++    data_t *data = NULL;
++    int ret = -1;
++
++    assert(str != NULL);
++
++    data = dict_get(xattr, (char *)name);
++    if (!data) {
++        *errnum = ENODATA;
++        goto out;
++    }
++
++    *str = GF_CALLOC(data->len + 1, sizeof(char), gf_common_mt_char);
++    if (!(*str)) {
++        *errnum = ENOMEM;
++        goto out;
++    }
++
++    memcpy(*str, data->data, sizeof(char) * (data->len));
++    return 0;
++
++out:
++    return ret;
++}
++
++int32_t
++__cs_get_dict_uuid(uuid_t uuid, dict_t *xattr, const char *name, int *errnum)
++{
++    data_t *data = NULL;
++    int ret = -1;
++
++    assert(uuid != NULL);
++
++    data = dict_get(xattr, (char *)name);
++    if (!data) {
++        *errnum = ENODATA;
++        goto out;
++    }
++
++    assert(data->len == sizeof(uuid_t));
++
++    gf_uuid_copy(uuid, (unsigned char *)data->data);
++    return 0;
++
++out:
++    return ret;
++}
++
+ int32_t
+ cs_fdctx_to_dict(xlator_t *this, fd_t *fd, dict_t *dict)
+ {
+@@ -1606,7 +2022,6 @@ cs_notify(xlator_t *this, int event, void *data, ...)
+ 
+ struct xlator_fops cs_fops = {
+     .stat = cs_stat,
+-    .readdirp = cs_readdirp,
+     .truncate = cs_truncate,
+     .seek = cs_seek,
+     .statfs = cs_statfs,
+@@ -1627,7 +2042,9 @@ struct xlator_fops cs_fops = {
+     .zerofill = cs_zerofill,
+ };
+ 
+-struct xlator_cbks cs_cbks = {};
++struct xlator_cbks cs_cbks = {
++    .forget = cs_forget,
++};
+ 
+ struct xlator_dumpops cs_dumpops = {
+     .fdctx_to_dict = cs_fdctx_to_dict,
+@@ -1647,6 +2064,15 @@ struct volume_options cs_options[] = {
+     {.key = {"cloudsync-storetype"},
+      .type = GF_OPTION_TYPE_STR,
+      .description = "Defines which remote store is enabled"},
++    {.key = {"cloudsync-remote-read"},
++     .type = GF_OPTION_TYPE_BOOL,
++     .description = "Defines a remote read fop when on"},
++    {.key = {"cloudsync-store-id"},
++     .type = GF_OPTION_TYPE_STR,
++     .description = "Defines a volume wide store id"},
++    {.key = {"cloudsync-product-id"},
++     .type = GF_OPTION_TYPE_STR,
++     .description = "Defines a volume wide product id"},
+     {.key = {NULL}},
+ };
+ 
+diff --git a/xlators/features/cloudsync/src/cloudsync.h b/xlators/features/cloudsync/src/cloudsync.h
+index dbdb207..0cb800a 100644
+--- a/xlators/features/cloudsync/src/cloudsync.h
++++ b/xlators/features/cloudsync/src/cloudsync.h
+@@ -19,6 +19,7 @@
+ #include "cloudsync-common.h"
+ #include "cloudsync-autogen-fops.h"
+ 
++#define ALIGN_SIZE 4096
+ #define CS_LOCK_DOMAIN "cs.protect.file.stat"
+ typedef struct cs_dlstore {
+     off_t off;
+@@ -29,6 +30,7 @@ typedef struct cs_dlstore {
+ } cs_dlstore;
+ 
+ typedef struct cs_inode_ctx {
++    cs_loc_xattr_t locxattr;
+     gf_cs_obj_state state;
+ } cs_inode_ctx_t;
+ 
+@@ -100,4 +102,22 @@ cs_truncate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+ int32_t
+ cs_resume_truncate(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                    off_t offset, dict_t *xattr_req);
++
++int32_t
++cs_readv_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
++             int32_t op_errno, struct iovec *vector, int32_t count,
++             struct iatt *stbuf, struct iobref *iobref, dict_t *xdata);
++int32_t
++cs_resume_readv(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
++                off_t offset, uint32_t flags, dict_t *xdata);
++int32_t
++cs_readv(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
++         off_t offset, uint32_t flags, dict_t *xdata);
++
++int
++cs_resume_remote_readv_postprocess(xlator_t *this, call_frame_t *frame,
++                                   inode_t *inode, off_t offset, size_t size,
++                                   uint32_t flags);
++int
++cs_serve_readv(call_frame_t *frame, off_t offset, size_t size, uint32_t flags);
+ #endif /* __CLOUDSYNC_H__ */
+diff --git a/xlators/mgmt/glusterd/src/glusterd-volume-set.c b/xlators/mgmt/glusterd/src/glusterd-volume-set.c
+index 4b32fb6..73abf37 100644
+--- a/xlators/mgmt/glusterd/src/glusterd-volume-set.c
++++ b/xlators/mgmt/glusterd/src/glusterd-volume-set.c
+@@ -3693,7 +3693,7 @@ struct volopt_map_entry glusterd_volopt_map[] = {
+      .op_version = GD_OP_VERSION_5_0,
+      .description = "enable/disable noatime option with ctime enabled.",
+      .flags = VOLOPT_FLAG_CLIENT_OPT | VOLOPT_FLAG_XLATOR_OPT},
+-    {.key = "feature.cloudsync-storetype",
++    {.key = "features.cloudsync-storetype",
+      .voltype = "features/cloudsync",
+      .op_version = GD_OP_VERSION_5_0,
+      .flags = VOLOPT_FLAG_CLIENT_OPT},
+@@ -3721,4 +3721,9 @@ struct volopt_map_entry glusterd_volopt_map[] = {
+      .validate_fn = validate_boolean,
+      .description = "option to enforce mandatory lock on a file",
+      .flags = VOLOPT_FLAG_XLATOR_OPT},
++    {.key = "features.cloudsync-remote-read",
++     .voltype = "features/cloudsync",
++     .value = "off",
++     .op_version = GD_OP_VERSION_6_0,
++     .flags = VOLOPT_FLAG_CLIENT_OPT},
+     {.key = NULL}};
+-- 
+1.8.3.1
+