path: root/0153-features-cloudsync-Added-some-new-functions.patch

From 90254e4ae9455fa0a126f83700978a9314eb79ea Mon Sep 17 00:00:00 2001
From: Anuradha Talur <atalur@commvault.com>
Date: Thu, 29 Nov 2018 12:54:21 -0800
Subject: [PATCH 153/169] features/cloudsync : Added some new functions

This patch contains the following changes:
1) Store ID info will now be stored in the inode ctx
2) Added new readv type where read is made directly
   from the remote store. This choice is made by
   volume set operation.
3) cs_forget() was missing. Added it.

backport of:https://review.gluster.org/#/c/glusterfs/+/21757/

> Change-Id: Ie3232b3d7ffb5313a03f011b0553b19793eedfa2
> fixes: bz#1642168
> Signed-off-by: Anuradha Talur <atalur@commvault.com>

Change-Id: I089e5a8c93049cf6bfabf011673796e38e78d7ee
Signed-off-by: Susant Palai <spalai@redhat.com>
Reviewed-on: https://code.engineering.redhat.com/gerrit/172192
Tested-by: RHGS Build Bot <nigelb@redhat.com>
Reviewed-by: Atin Mukherjee <amukherj@redhat.com>
---
 xlators/features/cloudsync/src/cloudsync-common.c  |  16 +
 xlators/features/cloudsync/src/cloudsync-common.h  |  35 ++
 xlators/features/cloudsync/src/cloudsync-fops-c.py |  12 +-
 .../features/cloudsync/src/cloudsync-mem-types.h   |   1 +
 xlators/features/cloudsync/src/cloudsync.c         | 600 ++++++++++++++++++---
 xlators/features/cloudsync/src/cloudsync.h         |  20 +
 xlators/mgmt/glusterd/src/glusterd-volume-set.c    |   7 +-
 7 files changed, 597 insertions(+), 94 deletions(-)

diff --git a/xlators/features/cloudsync/src/cloudsync-common.c b/xlators/features/cloudsync/src/cloudsync-common.c
index aee1f06..445a31b 100644
--- a/xlators/features/cloudsync/src/cloudsync-common.c
+++ b/xlators/features/cloudsync/src/cloudsync-common.c
@@ -11,6 +11,20 @@
 #include "cloudsync-common.h"
 
 void
+cs_xattrinfo_wipe(cs_local_t *local)
+{
+    if (local->xattrinfo.lxattr) {
+        if (local->xattrinfo.lxattr->file_path)
+            GF_FREE(local->xattrinfo.lxattr->file_path);
+
+        if (local->xattrinfo.lxattr->volname)
+            GF_FREE(local->xattrinfo.lxattr->volname);
+
+        GF_FREE(local->xattrinfo.lxattr);
+    }
+}
+
+void
 cs_local_wipe(xlator_t *this, cs_local_t *local)
 {
     if (!local)
@@ -40,5 +54,7 @@ cs_local_wipe(xlator_t *this, cs_local_t *local)
     if (local->remotepath)
         GF_FREE(local->remotepath);
 
+    cs_xattrinfo_wipe(local);
+
     mem_put(local);
 }
diff --git a/xlators/features/cloudsync/src/cloudsync-common.h b/xlators/features/cloudsync/src/cloudsync-common.h
index 7b3520c..11d2334 100644
--- a/xlators/features/cloudsync/src/cloudsync-common.h
+++ b/xlators/features/cloudsync/src/cloudsync-common.h
@@ -14,9 +14,23 @@
 #include <glusterfs/call-stub.h>
 #include <glusterfs/xlator.h>
 #include <glusterfs/syncop.h>
+#include <glusterfs/compat-errno.h>
 #include "cloudsync-mem-types.h"
 #include "cloudsync-messages.h"
 
+typedef struct cs_loc_xattr {
+    char *file_path;
+    uuid_t uuid;
+    uuid_t gfid;
+    char *volname;
+} cs_loc_xattr_t;
+
+typedef struct cs_size_xattr {
+    uint64_t size;
+    uint64_t blksize;
+    uint64_t blocks;
+} cs_size_xattr_t;
+
 typedef struct cs_local {
     loc_t loc;
     fd_t *fd;
@@ -34,10 +48,25 @@ typedef struct cs_local {
     int call_cnt;
     inode_t *inode;
     char *remotepath;
+
+    struct {
+        /* offset, flags and size are the information needed
+         * by read fop for remote read operation. These will be
+         * populated in cloudsync read fop, before being passed
+         * on to the plugin performing remote read.
+         */
+        off_t offset;
+        uint32_t flags;
+        size_t size;
+        cs_loc_xattr_t *lxattr;
+    } xattrinfo;
+
 } cs_local_t;
 
 typedef int (*fop_download_t)(call_frame_t *frame, void *config);
 
+typedef int (*fop_remote_read_t)(call_frame_t *, void *);
+
 typedef void *(*store_init)(xlator_t *this);
 
 typedef int (*store_reconfigure)(xlator_t *this, dict_t *options);
@@ -48,6 +77,7 @@ struct cs_remote_stores {
     char *name;                    /* store name */
     void *config;                  /* store related information */
     fop_download_t dlfop;          /* store specific download function */
+    fop_remote_read_t rdfop;       /* store specific read function */
     store_init init;               /* store init to initialize store config */
     store_reconfigure reconfigure; /* reconfigure store config */
     store_fini fini;
@@ -59,11 +89,15 @@ typedef struct cs_private {
     struct cs_remote_stores *stores;
     gf_boolean_t abortdl;
     pthread_spinlock_t lock;
+    gf_boolean_t remote_read;
 } cs_private_t;
 
 void
 cs_local_wipe(xlator_t *this, cs_local_t *local);
 
+void
+cs_xattrinfo_wipe(cs_local_t *local);
+
 #define CS_STACK_UNWIND(fop, frame, params...)                                 \
     do {                                                                       \
         cs_local_t *__local = NULL;                                            \
@@ -90,6 +124,7 @@ cs_local_wipe(xlator_t *this, cs_local_t *local);
 
 typedef struct store_methods {
     int (*fop_download)(call_frame_t *frame, void *config);
+    int (*fop_remote_read)(call_frame_t *, void *);
     /* return type should be the store config */
     void *(*fop_init)(xlator_t *this);
     int (*fop_reconfigure)(xlator_t *this, dict_t *options);
diff --git a/xlators/features/cloudsync/src/cloudsync-fops-c.py b/xlators/features/cloudsync/src/cloudsync-fops-c.py
index 3122bd3..a7a2201 100755
--- a/xlators/features/cloudsync/src/cloudsync-fops-c.py
+++ b/xlators/features/cloudsync/src/cloudsync-fops-c.py
@@ -137,15 +137,15 @@ cs_@NAME@_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         } else {
                                 __cs_inode_ctx_update (this, fd->inode, val);
                                 gf_msg (this->name, GF_LOG_INFO, 0, 0,
-                                        " state = %ld", val);
+                                        " state = %" PRIu64, val);
 
                                 if (local->call_cnt == 1 &&
                                     (val == GF_CS_REMOTE ||
                                      val == GF_CS_DOWNLOADING))  {
                                         gf_msg (this->name, GF_LOG_INFO, 0,
                                                 0, " will repair and download "
-                                                "the file, current state : %ld",
-                                                val);
+                                                "the file, current state : %"
+                                                PRIu64, val);
                                         goto repair;
                                 } else {
                                         gf_msg (this->name, GF_LOG_ERROR, 0, 0,
@@ -274,7 +274,7 @@ fd_ops = ['readv', 'writev', 'flush', 'fsync', 'fsyncdir', 'ftruncate',
 # These are the current actual lists used to generate the code
 
 # The following list contains fops which are fd based that modifies data
-fd_data_modify_op_fop_template = ['readv', 'writev', 'flush', 'fsync',
+fd_data_modify_op_fop_template = ['writev', 'flush', 'fsync',
                                   'ftruncate', 'rchecksum', 'fallocate',
                                   'discard', 'zerofill', 'seek']
 
@@ -284,8 +284,8 @@ loc_stat_op_fop_template = ['lookup', 'stat', 'discover', 'access', 'setattr',
                             'getattr']
 
 # These fops need a separate implementation
-special_fops = ['readdirp', 'statfs', 'setxattr', 'unlink', 'getxattr',
-                'truncate', 'fstat']
+special_fops = ['statfs', 'setxattr', 'unlink', 'getxattr',
+                'truncate', 'fstat', 'readv']
 
 def gen_defaults():
     for name in ops:
diff --git a/xlators/features/cloudsync/src/cloudsync-mem-types.h b/xlators/features/cloudsync/src/cloudsync-mem-types.h
index 9e6837a..2203464 100644
--- a/xlators/features/cloudsync/src/cloudsync-mem-types.h
+++ b/xlators/features/cloudsync/src/cloudsync-mem-types.h
@@ -16,6 +16,7 @@ enum cs_mem_types_ {
     gf_cs_mt_cs_private_t = gf_common_mt_end + 1,
     gf_cs_mt_cs_remote_stores_t,
     gf_cs_mt_cs_inode_ctx_t,
+    gf_cs_mt_cs_lxattr_t,
     gf_cs_mt_end
 };
 #endif /* __CLOUDSYNC_MEM_TYPES_H__ */
diff --git a/xlators/features/cloudsync/src/cloudsync.c b/xlators/features/cloudsync/src/cloudsync.c
index fbdcdf7..2240fc3 100644
--- a/xlators/features/cloudsync/src/cloudsync.c
+++ b/xlators/features/cloudsync/src/cloudsync.c
@@ -16,6 +16,7 @@
 #include <glusterfs/call-stub.h>
 #include "cloudsync-autogen-fops.h"
 
+#include <string.h>
 #include <dlfcn.h>
 
 void
@@ -72,6 +73,8 @@ cs_init(xlator_t *this)
 
     this->private = priv;
 
+    GF_OPTION_INIT("cloudsync-remote-read", priv->remote_read, bool, out);
+
     /* temp workaround. Should be configurable through glusterd*/
     per_vol = _gf_true;
 
@@ -135,6 +138,18 @@ cs_init(xlator_t *this)
 
         (void)dlerror();
 
+        if (priv->remote_read) {
+            priv->stores->rdfop = store_methods->fop_remote_read;
+            if (!priv->stores->rdfop) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, 0,
+                       "failed to get"
+                       " read fop %s",
+                       dlerror());
+                ret = -1;
+                goto out;
+            }
+        }
+
         priv->stores->dlfop = store_methods->fop_download;
         if (!priv->stores->dlfop) {
             gf_msg(this->name, GF_LOG_ERROR, 0, 0,
@@ -196,6 +211,22 @@ out:
     return ret;
 }
 
+int
+cs_forget(xlator_t *this, inode_t *inode)
+{
+    uint64_t ctx_int = 0;
+    cs_inode_ctx_t *ctx = NULL;
+
+    inode_ctx_del(inode, this, &ctx_int);
+    if (!ctx_int)
+        return 0;
+
+    ctx = (cs_inode_ctx_t *)(uintptr_t)ctx_int;
+
+    GF_FREE(ctx);
+    return 0;
+}
+
 void
 cs_fini(xlator_t *this)
 {
@@ -217,6 +248,9 @@ cs_reconfigure(xlator_t *this, dict_t *options)
         goto out;
     }
 
+    GF_OPTION_RECONF("cloudsync-remote-read", priv->remote_read, options, bool,
+                     out);
+
     /* needed only for per volume configuration*/
     ret = priv->stores->reconfigure(this, options);
 
@@ -242,59 +276,6 @@ out:
 }
 
 int32_t
-cs_readdirp_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
-                int32_t op_ret, int32_t op_errno, gf_dirent_t *entries,
-                dict_t *xdata)
-{
-    gf_dirent_t *tmp = NULL;
-    char *sxattr = NULL;
-    uint64_t ia_size = 0;
-    int ret = 0;
-
-    list_for_each_entry(tmp, &entries->list, list)
-    {
-        ret = dict_get_str(tmp->dict, GF_CS_OBJECT_SIZE, &sxattr);
-        if (ret) {
-            gf_msg_trace(this->name, 0, "size xattr found");
-            continue;
-        }
-
-        ia_size = atoll(sxattr);
-        tmp->d_stat.ia_size = ia_size;
-    }
-
-    STACK_UNWIND_STRICT(readdirp, frame, op_ret, op_errno, entries, xdata);
-    return 0;
-}
-
-int32_t
-cs_readdirp(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-            off_t off, dict_t *xdata)
-{
-    int ret = 0;
-    int op_errno = ENOMEM;
-
-    if (!xdata) {
-        xdata = dict_new();
-        if (!xdata) {
-            goto err;
-        }
-    }
-
-    ret = dict_set_int32(xdata, GF_CS_OBJECT_SIZE, 1);
-    if (ret) {
-        goto err;
-    }
-
-    STACK_WIND(frame, cs_readdirp_cbk, FIRST_CHILD(this),
-               FIRST_CHILD(this)->fops->readdirp, fd, size, off, xdata);
-    return 0;
-err:
-    STACK_UNWIND_STRICT(readdirp, frame, -1, op_errno, NULL, NULL);
-    return 0;
-}
-
-int32_t
 cs_truncate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
                 int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
                 struct iatt *postbuf, dict_t *xdata)
@@ -305,7 +286,6 @@ cs_truncate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
 
     local = frame->local;
 
-    /* Do we need lock here? */
     local->call_cnt++;
 
     if (op_ret == -1) {
@@ -320,13 +300,13 @@ cs_truncate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
                 goto unwind;
             } else {
                 __cs_inode_ctx_update(this, local->loc.inode, val);
-                gf_msg(this->name, GF_LOG_INFO, 0, 0, " state = %ld", val);
+                gf_msg(this->name, GF_LOG_INFO, 0, 0, " state = %" PRIu64, val);
 
                 if (local->call_cnt == 1 &&
                     (val == GF_CS_REMOTE || val == GF_CS_DOWNLOADING)) {
                     gf_msg(this->name, GF_LOG_WARNING, 0, 0,
                            "will repair and download "
-                           "the file, current state : %ld",
+                           "the file, current state : %" PRIu64,
                            val);
                     goto repair;
                 } else {
@@ -665,7 +645,7 @@ cs_fstat_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
     if (op_ret == 0) {
         ret = dict_get_uint64(xdata, GF_CS_OBJECT_STATUS, &val);
         if (!ret) {
-            gf_msg_debug(this->name, 0, "state %ld", val);
+            gf_msg_debug(this->name, 0, "state %" PRIu64, val);
             ret = __cs_inode_ctx_update(this, fd->inode, val);
             if (ret) {
                 gf_msg(this->name, GF_LOG_ERROR, 0, 0, "ctx update failed");
@@ -831,7 +811,7 @@ out:
     return 0;
 }
 
-void *
+int
 cs_download_task(void *arg)
 {
     call_frame_t *frame = NULL;
@@ -842,7 +822,6 @@ cs_download_task(void *arg)
     fd_t *fd = NULL;
     cs_local_t *local = NULL;
     dict_t *dict = NULL;
-    int *retval = NULL;
 
     frame = (call_frame_t *)arg;
 
@@ -850,13 +829,6 @@ cs_download_task(void *arg)
 
     priv = this->private;
 
-    retval = GF_CALLOC(1, sizeof(int), gf_common_mt_int);
-    if (!retval) {
-        gf_msg(this->name, GF_LOG_ERROR, 0, 0, "insufficient memory");
-        ret = -1;
-        goto out;
-    }
-
     if (!priv->stores) {
         gf_msg(this->name, GF_LOG_ERROR, 0, 0,
                "No remote store "
@@ -972,20 +944,13 @@ out:
         local->dlfd = NULL;
     }
 
-    if (retval) {
-        *retval = ret;
-        pthread_exit(retval);
-    } else {
-        pthread_exit(&ret);
-    }
+    return ret;
 }
 
 int
 cs_download(call_frame_t *frame)
 {
-    int *retval = NULL;
     int ret = 0;
-    pthread_t dthread;
     cs_local_t *local = NULL;
     xlator_t *this = NULL;
 
@@ -1000,16 +965,406 @@ cs_download(call_frame_t *frame)
         goto out;
     }
 
-    ret = gf_thread_create(&dthread, NULL, &cs_download_task, (void *)frame,
-                           "downloadthread");
+    ret = cs_download_task((void *)frame);
+out:
+    return ret;
+}
 
-    pthread_join(dthread, (void **)&retval);
+int
+cs_set_xattr_req(call_frame_t *frame)
+{
+    cs_local_t *local = NULL;
+    GF_UNUSED int ret = 0;
+
+    local = frame->local;
+
+    /* When remote reads are performed (i.e. reads on remote store),
+     * there needs to be a way to associate a file on gluster volume
+     * with its correspnding file on the remote store. In order to do
+     * that, a unique key can be maintained as an xattr
+     * (GF_CS_XATTR_ARCHIVE_UUID)on the stub file on gluster bricks.
+     * This xattr should be provided to the plugin to
+     * perform the read fop on the correct file. This assumes that the file
+     * hierarchy and name need not be the same on remote store as that of
+     * the gluster volume.
+     */
+    ret = dict_set_str(local->xattr_req, GF_CS_XATTR_ARCHIVE_UUID, "1");
+
+    return 0;
+}
 
-    ret = *retval;
+int
+cs_update_xattrs(call_frame_t *frame, dict_t *xdata)
+{
+    cs_local_t *local = NULL;
+    xlator_t *this = NULL;
+    int size = -1;
+    GF_UNUSED int ret = 0;
+
+    local = frame->local;
+    this = frame->this;
+
+    local->xattrinfo.lxattr = GF_CALLOC(1, sizeof(cs_loc_xattr_t),
+                                        gf_cs_mt_cs_lxattr_t);
+    if (!local->xattrinfo.lxattr) {
+        local->op_ret = -1;
+        local->op_errno = ENOMEM;
+        goto err;
+    }
+
+    gf_uuid_copy(local->xattrinfo.lxattr->gfid, local->loc.gfid);
+
+    if (local->remotepath) {
+        local->xattrinfo.lxattr->file_path = gf_strdup(local->remotepath);
+        if (!local->xattrinfo.lxattr->file_path) {
+            local->op_ret = -1;
+            local->op_errno = ENOMEM;
+            goto err;
+        }
+    }
+
+    ret = dict_get_gfuuid(xdata, GF_CS_XATTR_ARCHIVE_UUID,
+                          &(local->xattrinfo.lxattr->uuid));
+
+    if (ret) {
+        gf_uuid_clear(local->xattrinfo.lxattr->uuid);
+    }
+    size = strlen(this->name) - strlen("-cloudsync") + 1;
+    local->xattrinfo.lxattr->volname = GF_CALLOC(1, size, gf_common_mt_char);
+    if (!local->xattrinfo.lxattr->volname) {
+        local->op_ret = -1;
+        local->op_errno = ENOMEM;
+        goto err;
+    }
+    strncpy(local->xattrinfo.lxattr->volname, this->name, size - 1);
+    local->xattrinfo.lxattr->volname[size - 1] = '\0';
+
+    return 0;
+err:
+    cs_xattrinfo_wipe(local);
+    return -1;
+}
+
+int
+cs_serve_readv(call_frame_t *frame, off_t offset, size_t size, uint32_t flags)
+{
+    xlator_t *this = NULL;
+    cs_private_t *priv = NULL;
+    int ret = -1;
+    fd_t *fd = NULL;
+    cs_local_t *local = NULL;
+
+    local = frame->local;
+    this = frame->this;
+    priv = this->private;
+
+    if (!local->remotepath) {
+        ret = -1;
+        gf_msg(this->name, GF_LOG_ERROR, 0, 0,
+               "remote path not"
+               " available. Check posix logs to resolve");
+        goto out;
+    }
+
+    if (!priv->stores) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, 0,
+               "No remote store "
+               "plugins found");
+        ret = -1;
+        goto out;
+    }
+
+    if (local->fd) {
+        fd = fd_anonymous(local->fd->inode);
+    } else {
+        fd = fd_anonymous(local->loc.inode);
+    }
+
+    local->xattrinfo.size = size;
+    local->xattrinfo.offset = offset;
+    local->xattrinfo.flags = flags;
+
+    if (!fd) {
+        gf_msg("CS", GF_LOG_ERROR, 0, 0, "fd creation failed");
+        ret = -1;
+        goto out;
+    }
+
+    local->dlfd = fd;
+    local->dloffset = offset;
+
+    /*this calling method is for per volume setting */
+    ret = priv->stores->rdfop(frame, priv->stores->config);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, 0,
+               "read failed"
+               ", remotepath: %s",
+               local->remotepath);
+        ret = -1;
+        goto out;
+    } else {
+        gf_msg(this->name, GF_LOG_INFO, 0, 0,
+               "read success, path"
+               " : %s",
+               local->remotepath);
+    }
 
 out:
-    if (retval)
-        GF_FREE(retval);
+    if (fd) {
+        fd_unref(fd);
+        local->dlfd = NULL;
+    }
+    return ret;
+}
+
+int32_t
+cs_readv_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+             int32_t op_errno, struct iovec *vector, int32_t count,
+             struct iatt *stbuf, struct iobref *iobref, dict_t *xdata)
+{
+    cs_local_t *local = NULL;
+    int ret = 0;
+    uint64_t val = 0;
+    fd_t *fd = NULL;
+
+    local = frame->local;
+    fd = local->fd;
+
+    local->call_cnt++;
+
+    if (op_ret == -1) {
+        ret = dict_get_uint64(xdata, GF_CS_OBJECT_STATUS, &val);
+        if (ret == 0) {
+            if (val == GF_CS_ERROR) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, 0,
+                       "could not get file state, unwinding");
+                op_ret = -1;
+                op_errno = EIO;
+                goto unwind;
+            } else {
+                __cs_inode_ctx_update(this, fd->inode, val);
+                gf_msg(this->name, GF_LOG_INFO, 0, 0, " state = %" PRIu64, val);
+
+                if (local->call_cnt == 1 &&
+                    (val == GF_CS_REMOTE || val == GF_CS_DOWNLOADING)) {
+                    gf_msg(this->name, GF_LOG_INFO, 0, 0,
+                           " will read from remote : %" PRIu64, val);
+                    goto repair;
+                } else {
+                    gf_msg(this->name, GF_LOG_ERROR, 0, 0,
+                           "second readv, Unwinding");
+                    goto unwind;
+                }
+            }
+        } else {
+            gf_msg(this->name, GF_LOG_ERROR, 0, 0,
+                   "file state "
+                   "could not be figured, unwinding");
+            goto unwind;
+        }
+    } else {
+        /* successful readv => file is local */
+        __cs_inode_ctx_update(this, fd->inode, GF_CS_LOCAL);
+        gf_msg(this->name, GF_LOG_INFO, 0, 0,
+               "state : GF_CS_LOCAL"
+               ", readv successful");
+
+        goto unwind;
+    }
+
+repair:
+    ret = locate_and_execute(frame);
+    if (ret) {
+        goto unwind;
+    }
+
+    return 0;
+
+unwind:
+    CS_STACK_UNWIND(readv, frame, op_ret, op_errno, vector, count, stbuf,
+                    iobref, xdata);
+
+    return 0;
+}
+
+int32_t
+cs_resume_readv(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+                off_t offset, uint32_t flags, dict_t *xdata)
+{
+    int ret = 0;
+
+    ret = cs_resume_postprocess(this, frame, fd->inode);
+    if (ret) {
+        goto unwind;
+    }
+
+    cs_inodelk_unlock(frame);
+
+    STACK_WIND(frame, cs_readv_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->readv, fd, size, offset, flags, xdata);
+
+    return 0;
+
+unwind:
+    cs_inodelk_unlock(frame);
+
+    cs_common_cbk(frame);
+
+    return 0;
+}
+
+int32_t
+cs_resume_remote_readv(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                       size_t size, off_t offset, uint32_t flags, dict_t *xdata)
+{
+    int ret = 0;
+    cs_local_t *local = NULL;
+    gf_cs_obj_state state = -1;
+    cs_inode_ctx_t *ctx = NULL;
+
+    cs_inodelk_unlock(frame);
+
+    local = frame->local;
+    if (!local) {
+        ret = -1;
+        goto unwind;
+    }
+
+    __cs_inode_ctx_get(this, fd->inode, &ctx);
+
+    state = __cs_get_file_state(this, fd->inode, ctx);
+    if (state == GF_CS_ERROR) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, 0,
+               "status is GF_CS_ERROR."
+               " Aborting readv");
+        local->op_ret = -1;
+        local->op_errno = EREMOTE;
+        ret = -1;
+        goto unwind;
+    }
+
+    /* Serve readv from remote store only if it is remote. */
+    gf_msg_debug(this->name, 0, "status of file %s is %d",
+                 local->remotepath ? local->remotepath : "", state);
+
+    /* We will reach this condition if local inode ctx had REMOTE
+     * state when the control was in cs_readv but after stat
+     * we got an updated state saying that the file is LOCAL.
+     */
+    if (state == GF_CS_LOCAL) {
+        STACK_WIND(frame, cs_readv_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->readv, fd, size, offset, flags,
+                   xdata);
+    } else if (state == GF_CS_REMOTE) {
+        ret = cs_resume_remote_readv_postprocess(this, frame, fd->inode, offset,
+                                                 size, flags);
+        /* Failed to submit the remote readv fop to plugin */
+        if (ret) {
+            local->op_ret = -1;
+            local->op_errno = EREMOTE;
+            goto unwind;
+        }
+        /* When the file is in any other intermediate state,
+         * we should not perform remote reads.
+         */
+    } else {
+        local->op_ret = -1;
+        local->op_errno = EINVAL;
+        goto unwind;
+    }
+
+    return 0;
+
+unwind:
+    cs_common_cbk(frame);
+
+    return 0;
+}
+
+int32_t
+cs_readv(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+         off_t offset, uint32_t flags, dict_t *xdata)
+{
+    int op_errno = -1;
+    cs_local_t *local = NULL;
+    int ret = 0;
+    cs_inode_ctx_t *ctx = NULL;
+    gf_cs_obj_state state = -1;
+    cs_private_t *priv = NULL;
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(fd, err);
+
+    priv = this->private;
+
+    local = cs_local_init(this, frame, NULL, fd, GF_FOP_READ);
+    if (!local) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, 0, "local init failed");
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    __cs_inode_ctx_get(this, fd->inode, &ctx);
+
+    if (ctx)
+        state = __cs_get_file_state(this, fd->inode, ctx);
+    else
+        state = GF_CS_LOCAL;
+
+    local->xattr_req = xdata ? dict_ref(xdata) : (xdata = dict_new());
+
+    ret = dict_set_uint32(local->xattr_req, GF_CS_OBJECT_STATUS, 1);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, 0,
+               "dict_set failed key:"
+               " %s",
+               GF_CS_OBJECT_STATUS);
+        goto err;
+    }
+
+    if (priv->remote_read) {
+        local->stub = fop_readv_stub(frame, cs_resume_remote_readv, fd, size,
+                                     offset, flags, xdata);
+    } else {
+        local->stub = fop_readv_stub(frame, cs_resume_readv, fd, size, offset,
+                                     flags, xdata);
+    }
+    if (!local->stub) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, 0, "insufficient memory");
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    if (state == GF_CS_LOCAL) {
+        STACK_WIND(frame, cs_readv_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->readv, fd, size, offset, flags,
+                   xdata);
+    } else {
+        local->call_cnt++;
+        ret = locate_and_execute(frame);
+        if (ret) {
+            op_errno = ENOMEM;
+            goto err;
+        }
+    }
+
+    return 0;
+
+err:
+    CS_STACK_UNWIND(readv, frame, -1, op_errno, NULL, -1, NULL, NULL, NULL);
+
+    return 0;
+}
+
+int
+cs_resume_remote_readv_postprocess(xlator_t *this, call_frame_t *frame,
+                                   inode_t *inode, off_t offset, size_t size,
+                                   uint32_t flags)
+{
+    int ret = 0;
+
+    ret = cs_serve_readv(frame, offset, size, flags);
 
     return ret;
 }
@@ -1059,7 +1414,7 @@ cs_stat_check_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
                 goto err;
             } else {
                 ret = __cs_inode_ctx_update(this, inode, val);
-                gf_msg_debug(this->name, 0, "status : %lu", val);
+                gf_msg_debug(this->name, 0, "status : %" PRIu64, val);
                 if (ret) {
                     gf_msg(this->name, GF_LOG_ERROR, 0, 0, "ctx update failed");
                     local->op_ret = -1;
@@ -1087,6 +1442,10 @@ cs_stat_check_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
             gf_msg_debug(this->name, 0, "NULL filepath");
         }
 
+        ret = cs_update_xattrs(frame, xdata);
+        if (ret)
+            goto err;
+
         local->op_ret = 0;
         local->xattr_rsp = dict_ref(xdata);
         memcpy(&local->stbuf, stbuf, sizeof(struct iatt));
@@ -1121,6 +1480,8 @@ cs_do_stat_check(call_frame_t *main_frame)
         goto err;
     }
 
+    cs_set_xattr_req(main_frame);
+
     if (local->fd) {
         STACK_WIND(main_frame, cs_stat_check_cbk, FIRST_CHILD(this),
                    FIRST_CHILD(this)->fops->fstat, local->fd, local->xattr_req);
@@ -1177,6 +1538,10 @@ cs_common_cbk(call_frame_t *frame)
                             NULL, NULL, NULL);
             break;
 
+        case GF_FOP_TRUNCATE:
+            CS_STACK_UNWIND(truncate, frame, local->op_ret, local->op_errno,
+                            NULL, NULL, NULL);
+            break;
         default:
             break;
     }
@@ -1427,7 +1792,7 @@ __cs_inode_ctx_get(xlator_t *this, inode_t *inode, cs_inode_ctx_t **ctx)
     if (ret)
         *ctx = NULL;
     else
-        *ctx = (cs_inode_ctx_t *)ctxint;
+        *ctx = (cs_inode_ctx_t *)(uintptr_t)ctxint;
 
     return;
 }
@@ -1452,7 +1817,7 @@ __cs_inode_ctx_update(xlator_t *this, inode_t *inode, uint64_t val)
 
             ctx->state = val;
 
-            ctxint = (uint64_t)ctx;
+            ctxint = (uint64_t)(uintptr_t)ctx;
 
             ret = __inode_ctx_set(inode, this, &ctxint);
             if (ret) {
@@ -1460,7 +1825,7 @@ __cs_inode_ctx_update(xlator_t *this, inode_t *inode, uint64_t val)
                 goto out;
             }
         } else {
-            ctx = (cs_inode_ctx_t *)ctxint;
+            ctx = (cs_inode_ctx_t *)(uintptr_t)ctxint;
 
             ctx->state = val;
         }
@@ -1483,7 +1848,7 @@ cs_inode_ctx_reset(xlator_t *this, inode_t *inode)
         return 0;
     }
 
-    ctx = (cs_inode_ctx_t *)ctxint;
+    ctx = (cs_inode_ctx_t *)(uintptr_t)ctxint;
 
     GF_FREE(ctx);
     return 0;
@@ -1532,6 +1897,57 @@ cs_resume_postprocess(xlator_t *this, call_frame_t *frame, inode_t *inode)
 out:
     return ret;
 }
+
+int32_t
+__cs_get_dict_str(char **str, dict_t *xattr, const char *name, int *errnum)
+{
+    data_t *data = NULL;
+    int ret = -1;
+
+    assert(str != NULL);
+
+    data = dict_get(xattr, (char *)name);
+    if (!data) {
+        *errnum = ENODATA;
+        goto out;
+    }
+
+    *str = GF_CALLOC(data->len + 1, sizeof(char), gf_common_mt_char);
+    if (!(*str)) {
+        *errnum = ENOMEM;
+        goto out;
+    }
+
+    memcpy(*str, data->data, sizeof(char) * (data->len));
+    return 0;
+
+out:
+    return ret;
+}
+
+int32_t
+__cs_get_dict_uuid(uuid_t uuid, dict_t *xattr, const char *name, int *errnum)
+{
+    data_t *data = NULL;
+    int ret = -1;
+
+    assert(uuid != NULL);
+
+    data = dict_get(xattr, (char *)name);
+    if (!data) {
+        *errnum = ENODATA;
+        goto out;
+    }
+
+    assert(data->len == sizeof(uuid_t));
+
+    gf_uuid_copy(uuid, (unsigned char *)data->data);
+    return 0;
+
+out:
+    return ret;
+}
+
 int32_t
 cs_fdctx_to_dict(xlator_t *this, fd_t *fd, dict_t *dict)
 {
@@ -1606,7 +2022,6 @@ cs_notify(xlator_t *this, int event, void *data, ...)
 
 struct xlator_fops cs_fops = {
     .stat = cs_stat,
-    .readdirp = cs_readdirp,
     .truncate = cs_truncate,
     .seek = cs_seek,
     .statfs = cs_statfs,
@@ -1627,7 +2042,9 @@ struct xlator_fops cs_fops = {
     .zerofill = cs_zerofill,
 };
 
-struct xlator_cbks cs_cbks = {};
+struct xlator_cbks cs_cbks = {
+    .forget = cs_forget,
+};
 
 struct xlator_dumpops cs_dumpops = {
     .fdctx_to_dict = cs_fdctx_to_dict,
@@ -1647,6 +2064,15 @@ struct volume_options cs_options[] = {
     {.key = {"cloudsync-storetype"},
      .type = GF_OPTION_TYPE_STR,
      .description = "Defines which remote store is enabled"},
+    {.key = {"cloudsync-remote-read"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .description = "Defines a remote read fop when on"},
+    {.key = {"cloudsync-store-id"},
+     .type = GF_OPTION_TYPE_STR,
+     .description = "Defines a volume wide store id"},
+    {.key = {"cloudsync-product-id"},
+     .type = GF_OPTION_TYPE_STR,
+     .description = "Defines a volume wide product id"},
     {.key = {NULL}},
 };
 
diff --git a/xlators/features/cloudsync/src/cloudsync.h b/xlators/features/cloudsync/src/cloudsync.h
index dbdb207..0cb800a 100644
--- a/xlators/features/cloudsync/src/cloudsync.h
+++ b/xlators/features/cloudsync/src/cloudsync.h
@@ -19,6 +19,7 @@
 #include "cloudsync-common.h"
 #include "cloudsync-autogen-fops.h"
 
+#define ALIGN_SIZE 4096
 #define CS_LOCK_DOMAIN "cs.protect.file.stat"
 typedef struct cs_dlstore {
     off_t off;
@@ -29,6 +30,7 @@ typedef struct cs_dlstore {
 } cs_dlstore;
 
 typedef struct cs_inode_ctx {
+    cs_loc_xattr_t locxattr;
     gf_cs_obj_state state;
 } cs_inode_ctx_t;
 
@@ -100,4 +102,22 @@ cs_truncate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
 int32_t
 cs_resume_truncate(call_frame_t *frame, xlator_t *this, loc_t *loc,
                    off_t offset, dict_t *xattr_req);
+
+int32_t
+cs_readv_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+             int32_t op_errno, struct iovec *vector, int32_t count,
+             struct iatt *stbuf, struct iobref *iobref, dict_t *xdata);
+int32_t
+cs_resume_readv(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+                off_t offset, uint32_t flags, dict_t *xdata);
+int32_t
+cs_readv(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+         off_t offset, uint32_t flags, dict_t *xdata);
+
+int
+cs_resume_remote_readv_postprocess(xlator_t *this, call_frame_t *frame,
+                                   inode_t *inode, off_t offset, size_t size,
+                                   uint32_t flags);
+int
+cs_serve_readv(call_frame_t *frame, off_t offset, size_t size, uint32_t flags);
 #endif /* __CLOUDSYNC_H__ */
diff --git a/xlators/mgmt/glusterd/src/glusterd-volume-set.c b/xlators/mgmt/glusterd/src/glusterd-volume-set.c
index 4b32fb6..73abf37 100644
--- a/xlators/mgmt/glusterd/src/glusterd-volume-set.c
+++ b/xlators/mgmt/glusterd/src/glusterd-volume-set.c
@@ -3693,7 +3693,7 @@ struct volopt_map_entry glusterd_volopt_map[] = {
      .op_version = GD_OP_VERSION_5_0,
      .description = "enable/disable noatime option with ctime enabled.",
      .flags = VOLOPT_FLAG_CLIENT_OPT | VOLOPT_FLAG_XLATOR_OPT},
-    {.key = "feature.cloudsync-storetype",
+    {.key = "features.cloudsync-storetype",
      .voltype = "features/cloudsync",
      .op_version = GD_OP_VERSION_5_0,
      .flags = VOLOPT_FLAG_CLIENT_OPT},
@@ -3721,4 +3721,9 @@ struct volopt_map_entry glusterd_volopt_map[] = {
      .validate_fn = validate_boolean,
      .description = "option to enforce mandatory lock on a file",
      .flags = VOLOPT_FLAG_XLATOR_OPT},
+    {.key = "features.cloudsync-remote-read",
+     .voltype = "features/cloudsync",
+     .value = "off",
+     .op_version = GD_OP_VERSION_6_0,
+     .flags = VOLOPT_FLAG_CLIENT_OPT},
     {.key = NULL}};
-- 
1.8.3.1