From 762c6c0c4c8618fc95aa25696345e369681d0465 Mon Sep 17 00:00:00 2001 From: Yang Yanchao Date: Fri, 25 Aug 2023 18:39:54 +0800 Subject: [PATCH] use ioctl instead of syscall --- Makefile.am | 3 ++- include/gmem_common.h | 16 ++++++------ include/libgmem.h | 24 +++++++++++------ src/ascend/gmem_ascend.c | 22 ++++------------ src/ascend/gmem_ascend.h | 35 ------------------------- src/init.c | 56 ++++++++++++++++++++++++++++++++++++++++ src/libgmem.c | 18 +++++++------ 7 files changed, 97 insertions(+), 77 deletions(-) delete mode 100644 src/ascend/gmem_ascend.h create mode 100644 src/init.c diff --git a/Makefile.am b/Makefile.am index e18c2fc..06b1435 100644 --- a/Makefile.am +++ b/Makefile.am @@ -23,7 +23,8 @@ include_HEADERS = include/libgmem.h lib_LTLIBRARIES = libgmem.la libgmem_la_SOURCES = \ - src/libgmem.c + src/libgmem.c \ + src/init.c libgmem_la_CFLAGS = $(AM_CFLAGS) \ -I$(top_srcdir)/include diff --git a/include/gmem_common.h b/include/gmem_common.h index 994f953..a400b37 100644 --- a/include/gmem_common.h +++ b/include/gmem_common.h @@ -13,22 +13,22 @@ #ifndef _GMEM_WAPPER_H_ #define _GMEM_WAPPER_H_ +#include #include -struct gm_msg { - int behavior; - unsigned long addr; - size_t size; - int hnid; -}; -typedef struct gm_msg *gm_msg_t; - typedef struct { int (*FreeEager)(void *userData, void *stream); int (*Prefetch)(void *userData, void *stream); int (*GetNumaId)(void); } gmem_semantics; extern gmem_semantics gmemSemantics; +extern int gmem_fd; + +typedef struct hmadvise_arg gm_msg; +typedef gm_msg *gm_msg_t; + +void init_device(void); +void destory_device(void); int gmemAdvise(void *userData); #endif diff --git a/include/libgmem.h b/include/libgmem.h index ea313a3..4cd918b 100644 --- a/include/libgmem.h +++ b/include/libgmem.h @@ -20,14 +20,22 @@ __BEGIN_DECLS /* * TODO: Remove these "ifndef" after kernel upgrade */ -/* - * __NR_madvise 441 - */ -#ifndef SYS_hmdvise -# ifndef __NR_hmadvise -# define __NR_hmadvise 441 -# endif -# define SYS_hmadvise __NR_hmadvise +#include +struct hmadvise_arg { + int hnid; + unsigned long start; + size_t len_in; + int behavior; +}; +struct gmem_hnid_arg { + int *hnuma_id; +}; +#ifndef GMEM_GET_HNUMA_ID + #define GMEM_GET_HNUMA_ID _IOW(0x55, 1, struct gmem_hnid_arg) +#endif + +#ifndef GMEM_MADVISE + #define GMEM_MADVISE _IOW(0x55, 2, struct hmadvise_arg) #endif #ifndef MADV_PREFETCH diff --git a/src/ascend/gmem_ascend.c b/src/ascend/gmem_ascend.c index be54813..6181c07 100644 --- a/src/ascend/gmem_ascend.c +++ b/src/ascend/gmem_ascend.c @@ -19,13 +19,8 @@ #include -#include "gmem_ascend.h" - #define LIB_ASCENDCL_PATH "/usr/local/Ascend/ascend-toolkit/latest/aarch64-linux/lib64/libascendcl.so" -void init_gmemSemantics(void) __attribute__ ((constructor)); -void exit_gmemSemantics(void) __attribute__ ((destructor)); - typedef void (*Callback_fn)(void *userData); typedef int (*ACLRT_LAUNCH_CB_FUNC)(Callback_fn fn, void *userData, int vlockType, void *stream); ACLRT_LAUNCH_CB_FUNC aclrtLaunchCallback_fn; @@ -43,26 +38,19 @@ int ascend_prefech(void *userData, void *stream) int ascend_numaid(void) { - int ret, id, fd; - struct rpg_hnid_arg arg; - - fd = open("/dev/remote_pager", O_RDWR); - if (fd == -1) { - perror("Get davinci_manager fd failed"); - return -ENXIO; - } + int ret, id; + struct gmem_hnid_arg arg; arg.hnuma_id = &id; - ret = ioctl(fd, RPG_GET_HNUMA_ID, &arg); + ret = ioctl(gmem_fd, GMEM_GET_HNUMA_ID, &arg); if (ret < 0) { perror("Get hnuma id ioctl failed."); return ret; } - close(fd); return id; } -void init_gmemSemantics() +void init_device() { handle = dlopen(LIB_ASCENDCL_PATH, RTLD_LAZY); if (!handle) { @@ -76,7 +64,7 @@ void init_gmemSemantics() gmemSemantics.GetNumaId = ascend_numaid; } -void exit_gmemSemantics() +void destory_device() { dlclose(handle); } diff --git a/src/ascend/gmem_ascend.h b/src/ascend/gmem_ascend.h deleted file mode 100644 index 5348c2e..0000000 --- a/src/ascend/gmem_ascend.h +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright (c) 2023 yangyanchao6@huawei.com  - * libgmem is licensed under Mulan PSL v2. - * You can use this software according to the terms and conditions of the Mulan PSL v2. - * You may obtain a copy of Mulan PSL v2 at: - *          http://license.coscl.org.cn/MulanPSL2 - * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, - * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, - * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. - * See the Mulan PSL v2 for more details. - */ - -#ifndef _GMEM_ASCEND_H_ -#define _GMEM_ASCEND_H_ - -/* - * These definitions should be placed in the kernel, - * which is a temporary implementation here. - */ - -#define RPG_MAGIC 0x55 - -enum _RPG_IOCTL_CMD { - _RPG_GET_HNUMA_ID = 1, - _RPG_IOC_MAX_NR -}; - -struct rpg_hnid_arg { - int *hnuma_id; -}; - -#define RPG_GET_HNUMA_ID \ - _IOW(RPG_MAGIC, _RPG_GET_HNUMA_ID, struct rpg_hnid_arg) - -#endif diff --git a/src/init.c b/src/init.c new file mode 100644 index 0000000..08b2800 --- /dev/null +++ b/src/init.c @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2023 yangyanchao6@huawei.com  + * libgmem is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + *          http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +#include +#include +#include +#include +#include + +#include +#include + +int gmem_fd; + +int init_libgmem(void) __attribute__ ((constructor)); +void exit_libgmem(void) __attribute__ ((destructor)); + +int get_gmem_fd() +{ + gmem_fd = open("/dev/gmem", O_RDWR); + if (gmem_fd == -1) { + perror("Get gmem fd failed"); + return -ENXIO; + } + return 0; +} + +void release_gmem_fd() +{ + close(gmem_fd); +} + +int init_libgmem(void) +{ + if (get_gmem_fd()) { + return -ENXIO; + } + init_device(); + return 0; +} + +void exit_libgmem(void) +{ + release_gmem_fd(); + destory_device(); +} + diff --git a/src/libgmem.c b/src/libgmem.c index a1a1640..9a34de0 100644 --- a/src/libgmem.c +++ b/src/libgmem.c @@ -23,17 +23,19 @@ gmem_semantics gmemSemantics; static void mix_userData(gm_msg_t userData, unsigned long addr, size_t length, int hnid, int behavior) { userData->behavior = behavior; - userData->addr = addr; - userData->size = length; + userData->start = addr; + userData->len_in = length; userData->hnid = hnid; } int gmemAdvise(void *userData) { - gm_msg_t msg = (gm_msg_t)userData; - int ret = syscall(SYS_hmadvise, msg->hnid, msg->addr, msg->size, msg->behavior); + int ret; + gm_msg msg = *(gm_msg_t)userData; + + ret = ioctl(gmem_fd, GMEM_MADVISE, msg); if(ret) { - printf("hmadvise failed: addr:%lx, size:%lx, behavior:%d, hnid:%d\n", msg->addr, msg->size, msg->behavior, msg->hnid); + printf("hmadvise failed: addr:%lx, size:%lx, behavior:%d, hnid:%d\n", msg.start, msg.len_in, msg.behavior, msg.hnid); } return ret; } @@ -41,7 +43,7 @@ int gmemAdvise(void *userData) int gmemFreeEager(unsigned long addr, size_t length, void *stream) { int ret = -1; - gm_msg_t userData = (gm_msg_t)malloc(sizeof(struct gm_msg)); + gm_msg_t userData = (gm_msg_t)malloc(sizeof(gm_msg)); mix_userData(userData, addr, length, -1, MADV_DONTNEED); if (!stream) { ret = gmemAdvise(userData); @@ -55,12 +57,12 @@ int gmemFreeEager(unsigned long addr, size_t length, void *stream) int gmemPrefetch(unsigned long addr, size_t length, void *stream) { int ret = -1; - gm_msg_t userData = (gm_msg_t)malloc(sizeof(struct gm_msg)); + gm_msg_t userData = (gm_msg_t)malloc(sizeof(gm_msg)); mix_userData(userData, addr, length, gmemGetNumaId(), MADV_PREFETCH); if (!stream) { ret = gmemAdvise(userData); } else if(gmemSemantics.Prefetch != NULL) { - ret = gmemSemantics.FreeEager(userData,stream); + ret = gmemSemantics.Prefetch(userData, stream); } free(userData); return ret; -- 2.41.0.windows.3