From ce69c916b2c1638515e85c4d08cbcf89601de336 Mon Sep 17 00:00:00 2001 From: Liangbin Lian Date: Wed, 12 Feb 2025 12:05:50 +0800 Subject: [PATCH] target/rk35xx: rknpu update to 0.9.8 --- .../rk35xx/files/drivers/rknpu/Makefile | 2 + .../drivers/rknpu/include/rknpu_devfreq.h | 46 + .../files/drivers/rknpu/include/rknpu_drv.h | 70 +- .../files/drivers/rknpu/include/rknpu_gem.h | 24 +- .../files/drivers/rknpu/include/rknpu_ioctl.h | 62 +- .../files/drivers/rknpu/include/rknpu_iommu.h | 60 + .../files/drivers/rknpu/include/rknpu_job.h | 9 +- .../files/drivers/rknpu/include/rknpu_mem.h | 7 +- .../files/drivers/rknpu/include/rknpu_mm.h | 19 - .../files/drivers/rknpu/rknpu_debugger.c | 14 +- .../files/drivers/rknpu/rknpu_devfreq.c | 807 ++++++++++ .../rk35xx/files/drivers/rknpu/rknpu_drv.c | 1302 ++++++----------- .../rk35xx/files/drivers/rknpu/rknpu_gem.c | 787 +++++++--- .../rk35xx/files/drivers/rknpu/rknpu_iommu.c | 619 ++++++++ .../rk35xx/files/drivers/rknpu/rknpu_job.c | 439 ++++-- .../rk35xx/files/drivers/rknpu/rknpu_mem.c | 127 +- .../rk35xx/files/drivers/rknpu/rknpu_mm.c | 51 - .../rk35xx/files/drivers/rknpu/rknpu_reset.c | 64 +- .../1027-export-iommu_setup_dma_ops.patch | 24 + .../1028-drivers-rknpu-backport.patch | 206 +++ 20 files changed, 3406 insertions(+), 1333 deletions(-) create mode 100644 target/linux/rockchip/rk35xx/files/drivers/rknpu/include/rknpu_devfreq.h create mode 100644 target/linux/rockchip/rk35xx/files/drivers/rknpu/include/rknpu_iommu.h create mode 100644 target/linux/rockchip/rk35xx/files/drivers/rknpu/rknpu_devfreq.c create mode 100644 target/linux/rockchip/rk35xx/files/drivers/rknpu/rknpu_iommu.c create mode 100644 target/linux/rockchip/rk35xx/patches-5.10/1027-export-iommu_setup_dma_ops.patch create mode 100644 target/linux/rockchip/rk35xx/patches-5.10/1028-drivers-rknpu-backport.patch diff --git a/target/linux/rockchip/rk35xx/files/drivers/rknpu/Makefile b/target/linux/rockchip/rk35xx/files/drivers/rknpu/Makefile index 41dacc9315..d73563dbaf 100644 --- a/target/linux/rockchip/rk35xx/files/drivers/rknpu/Makefile +++ b/target/linux/rockchip/rk35xx/files/drivers/rknpu/Makefile @@ -9,6 +9,8 @@ rknpu-y += rknpu_drv.o rknpu-y += rknpu_reset.o rknpu-y += rknpu_job.o rknpu-y += rknpu_debugger.o +rknpu-y += rknpu_iommu.o +rknpu-$(CONFIG_PM_DEVFREQ) += rknpu_devfreq.o rknpu-$(CONFIG_ROCKCHIP_RKNPU_SRAM) += rknpu_mm.o rknpu-$(CONFIG_ROCKCHIP_RKNPU_FENCE) += rknpu_fence.o rknpu-$(CONFIG_ROCKCHIP_RKNPU_DRM_GEM) += rknpu_gem.o diff --git a/target/linux/rockchip/rk35xx/files/drivers/rknpu/include/rknpu_devfreq.h b/target/linux/rockchip/rk35xx/files/drivers/rknpu/include/rknpu_devfreq.h new file mode 100644 index 0000000000..3edeb1fe4a --- /dev/null +++ b/target/linux/rockchip/rk35xx/files/drivers/rknpu/include/rknpu_devfreq.h @@ -0,0 +1,46 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) Rockchip Electronics Co.Ltd + * Author: Finley Xiao + */ + +#ifndef __LINUX_RKNPU_DEVFREQ_H +#define __LINUX_RKNPU_DEVFREQ_H + +#ifdef CONFIG_PM_DEVFREQ +void rknpu_devfreq_lock(struct rknpu_device *rknpu_dev); +void rknpu_devfreq_unlock(struct rknpu_device *rknpu_dev); +int rknpu_devfreq_init(struct rknpu_device *rknpu_dev); +void rknpu_devfreq_remove(struct rknpu_device *rknpu_dev); +int rknpu_devfreq_runtime_suspend(struct device *dev); +int rknpu_devfreq_runtime_resume(struct device *dev); +#else +static inline int rknpu_devfreq_init(struct rknpu_device *rknpu_dev) +{ + return -EOPNOTSUPP; +} + +static inline void rknpu_devfreq_remove(struct rknpu_device *rknpu_dev) +{ +} + +static inline void rknpu_devfreq_lock(struct rknpu_device *rknpu_dev) +{ +} + +static inline void rknpu_devfreq_unlock(struct rknpu_device *rknpu_dev) +{ +} + +static inline int rknpu_devfreq_runtime_suspend(struct device *dev) +{ + return -EOPNOTSUPP; +} + +static inline int rknpu_devfreq_runtime_resume(struct device *dev) +{ + return -EOPNOTSUPP; +} +#endif /* CONFIG_PM_DEVFREQ */ + +#endif /* __LINUX_RKNPU_DEVFREQ_H_ */ diff --git a/target/linux/rockchip/rk35xx/files/drivers/rknpu/include/rknpu_drv.h b/target/linux/rockchip/rk35xx/files/drivers/rknpu/include/rknpu_drv.h index 67da03cb63..90e66d68b7 100644 --- a/target/linux/rockchip/rk35xx/files/drivers/rknpu/include/rknpu_drv.h +++ b/target/linux/rockchip/rk35xx/files/drivers/rknpu/include/rknpu_drv.h @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -17,11 +18,9 @@ #include #include -#ifndef FPGA_PLATFORM -#if KERNEL_VERSION(5, 10, 0) <= LINUX_VERSION_CODE #include -#endif -#endif +#include +#include #include "rknpu_job.h" #include "rknpu_fence.h" @@ -30,10 +29,10 @@ #define DRIVER_NAME "rknpu" #define DRIVER_DESC "RKNPU driver" -#define DRIVER_DATE "20230202" +#define DRIVER_DATE "20240828" #define DRIVER_MAJOR 0 -#define DRIVER_MINOR 8 -#define DRIVER_PATCHLEVEL 5 +#define DRIVER_MINOR 9 +#define DRIVER_PATCHLEVEL 8 #define LOG_TAG "RKNPU" @@ -54,9 +53,19 @@ #define LOG_DEV_DEBUG(dev, fmt, args...) dev_dbg(dev, LOG_TAG ": " fmt, ##args) #define LOG_DEV_ERROR(dev, fmt, args...) dev_err(dev, LOG_TAG ": " fmt, ##args) -struct rknpu_reset_data { - const char *srst_a_name; - const char *srst_h_name; +#define RKNPU_MAX_IOMMU_DOMAIN_NUM 16 +#define RKNPU_CACHE_SG_TABLE_NUM 2 + +struct rknpu_irqs_data { + const char *name; + irqreturn_t (*irq_hdl)(int irq, void *ctx); +}; + +struct rknpu_amount_data { + uint16_t offset_clr_all; + uint16_t offset_dt_wr; + uint16_t offset_dt_rd; + uint16_t offset_wt_rd; }; struct rknpu_config { @@ -68,16 +77,21 @@ struct rknpu_config { __u32 pc_task_number_mask; __u32 pc_task_status_offset; __u32 pc_dma_ctrl; - __u32 bw_enable; const struct rknpu_irqs_data *irqs; - const struct rknpu_reset_data *resets; int num_irqs; - int num_resets; + __u64 nbuf_phyaddr; + __u64 nbuf_size; + __u64 max_submit_number; + __u32 core_mask; + const struct rknpu_amount_data *amount_top; + const struct rknpu_amount_data *amount_core; + void (*state_init)(struct rknpu_device *rknpu_dev); + int (*cache_sgt_init)(struct rknpu_device *rknpu_dev); }; struct rknpu_timer { - __u32 busy_time; - __u32 busy_time_record; + ktime_t busy_time; + ktime_t total_busy_time; }; struct rknpu_subcore_data { @@ -99,6 +113,7 @@ struct rknpu_device { void __iomem *base[RKNPU_MAX_CORES]; struct device *dev; #ifdef CONFIG_ROCKCHIP_RKNPU_DRM_GEM + struct device *fake_dev; struct drm_device *drm_dev; #endif #ifdef CONFIG_ROCKCHIP_RKNPU_DMA_HEAP @@ -110,13 +125,14 @@ struct rknpu_device { spinlock_t irq_lock; struct mutex power_lock; struct mutex reset_lock; + struct mutex domain_lock; struct rknpu_subcore_data subcore_datas[RKNPU_MAX_CORES]; const struct rknpu_config *config; void __iomem *bw_priority_base; struct rknpu_fence_context *fence_ctx; bool iommu_en; - struct reset_control *srst_a[RKNPU_MAX_CORES]; - struct reset_control *srst_h[RKNPU_MAX_CORES]; + struct reset_control **srsts; + int num_srsts; struct clk_bulk_data *clks; int num_clks; struct regulator *vdd; @@ -126,11 +142,7 @@ struct rknpu_device { struct thermal_cooling_device *devfreq_cooling; struct devfreq *devfreq; unsigned long ondemand_freq; -#ifndef FPGA_PLATFORM -#if KERNEL_VERSION(5, 10, 0) <= LINUX_VERSION_CODE struct rockchip_opp_info opp_info; -#endif -#endif unsigned long current_freq; unsigned long current_volt; int bypass_irq_handler; @@ -149,13 +161,29 @@ struct rknpu_device { ktime_t kt; phys_addr_t sram_start; phys_addr_t sram_end; + phys_addr_t nbuf_start; + phys_addr_t nbuf_end; uint32_t sram_size; + uint32_t nbuf_size; void __iomem *sram_base_io; + void __iomem *nbuf_base_io; struct rknpu_mm *sram_mm; unsigned long power_put_delay; + struct iommu_group *iommu_group; + int iommu_domain_num; + int iommu_domain_id; + struct iommu_domain *iommu_domains[RKNPU_MAX_IOMMU_DOMAIN_NUM]; + struct sg_table *cache_sgt[RKNPU_CACHE_SG_TABLE_NUM]; + atomic_t iommu_domain_refcount; +}; + +struct rknpu_session { + struct rknpu_device *rknpu_dev; + struct list_head list; }; int rknpu_power_get(struct rknpu_device *rknpu_dev); int rknpu_power_put(struct rknpu_device *rknpu_dev); +int rknpu_power_put_delay(struct rknpu_device *rknpu_dev); #endif /* __LINUX_RKNPU_DRV_H_ */ diff --git a/target/linux/rockchip/rk35xx/files/drivers/rknpu/include/rknpu_gem.h b/target/linux/rockchip/rk35xx/files/drivers/rknpu/include/rknpu_gem.h index 954586607b..17f9221057 100644 --- a/target/linux/rockchip/rk35xx/files/drivers/rknpu/include/rknpu_gem.h +++ b/target/linux/rockchip/rk35xx/files/drivers/rknpu/include/rknpu_gem.h @@ -48,6 +48,7 @@ struct rknpu_gem_object { unsigned int flags; unsigned long size; unsigned long sram_size; + unsigned long nbuf_size; struct rknpu_mm_obj *sram_obj; dma_addr_t iova_start; unsigned long iova_size; @@ -59,13 +60,21 @@ struct rknpu_gem_object { struct page **pages; struct sg_table *sgt; struct drm_mm_node mm_node; + int iommu_domain_id; + unsigned int core_mask; + unsigned int cache_with_sgt; +}; + +enum rknpu_cache_type { + RKNPU_CACHE_SRAM = 1 << 0, + RKNPU_CACHE_NBUF = 1 << 1, }; /* create a new buffer with gem object */ -struct rknpu_gem_object *rknpu_gem_object_create(struct drm_device *dev, - unsigned int flags, - unsigned long size, - unsigned long sram_size); +struct rknpu_gem_object * +rknpu_gem_object_create(struct drm_device *dev, unsigned int flags, + unsigned long size, unsigned long sram_size, + int iommu_domain_id, unsigned int core_mask); /* destroy a buffer with gem object */ void rknpu_gem_object_destroy(struct rknpu_gem_object *rknpu_obj); @@ -157,6 +166,8 @@ int rknpu_gem_fault(struct vm_fault *vmf); int rknpu_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf); #endif +int rknpu_gem_mmap_obj(struct drm_gem_object *obj, struct vm_area_struct *vma); + /* set vm_flags and we can change the vm attribute to other one at here. */ int rknpu_gem_mmap(struct file *filp, struct vm_area_struct *vma); @@ -170,8 +181,13 @@ struct drm_gem_object * rknpu_gem_prime_import_sg_table(struct drm_device *dev, struct dma_buf_attachment *attach, struct sg_table *sgt); +#if KERNEL_VERSION(6, 1, 0) > LINUX_VERSION_CODE void *rknpu_gem_prime_vmap(struct drm_gem_object *obj); void rknpu_gem_prime_vunmap(struct drm_gem_object *obj, void *vaddr); +#else +int rknpu_gem_prime_vmap(struct drm_gem_object *obj, struct iosys_map *map); +void rknpu_gem_prime_vunmap(struct drm_gem_object *obj, struct iosys_map *map); +#endif int rknpu_gem_prime_mmap(struct drm_gem_object *obj, struct vm_area_struct *vma); diff --git a/target/linux/rockchip/rk35xx/files/drivers/rknpu/include/rknpu_ioctl.h b/target/linux/rockchip/rk35xx/files/drivers/rknpu/include/rknpu_ioctl.h index fc7225fb7b..98c544464d 100644 --- a/target/linux/rockchip/rk35xx/files/drivers/rknpu/include/rknpu_ioctl.h +++ b/target/linux/rockchip/rk35xx/files/drivers/rknpu/include/rknpu_ioctl.h @@ -31,11 +31,6 @@ #define RKNPU_OFFSET_INT_STATUS 0x28 #define RKNPU_OFFSET_INT_RAW_STATUS 0x2c -#define RKNPU_OFFSET_CLR_ALL_RW_AMOUNT 0x8010 -#define RKNPU_OFFSET_DT_WR_AMOUNT 0x8034 -#define RKNPU_OFFSET_DT_RD_AMOUNT 0x8038 -#define RKNPU_OFFSET_WT_RD_AMOUNT 0x803c - #define RKNPU_OFFSET_ENABLE_MASK 0xf008 #define RKNPU_INT_CLEAR 0x1ffff @@ -44,10 +39,10 @@ #define RKNPU_STR_HELPER(x) #x -#define RKNPU_GET_DRV_VERSION_STRING(MAJOR, MINOR, PATCHLEVEL) \ - RKNPU_STR_HELPER(MAJOR) \ +#define RKNPU_GET_DRV_VERSION_STRING(MAJOR, MINOR, PATCHLEVEL) \ + RKNPU_STR_HELPER(MAJOR) \ "." RKNPU_STR_HELPER(MINOR) "." RKNPU_STR_HELPER(PATCHLEVEL) -#define RKNPU_GET_DRV_VERSION_CODE(MAJOR, MINOR, PATCHLEVEL) \ +#define RKNPU_GET_DRV_VERSION_CODE(MAJOR, MINOR, PATCHLEVEL) \ (MAJOR * 10000 + MINOR * 100 + PATCHLEVEL) #define RKNPU_GET_DRV_VERSION_MAJOR(CODE) (CODE / 10000) #define RKNPU_GET_DRV_VERSION_MINOR(CODE) ((CODE % 10000) / 100) @@ -67,29 +62,34 @@ enum e_rknpu_mem_type { RKNPU_MEM_WRITE_COMBINE = 1 << 2, /* dma attr kernel mapping */ RKNPU_MEM_KERNEL_MAPPING = 1 << 3, - /* iommu mapping */ + /* IOMMU mapping */ RKNPU_MEM_IOMMU = 1 << 4, /* zero mapping */ RKNPU_MEM_ZEROING = 1 << 5, /* allocate secure buffer */ RKNPU_MEM_SECURE = 1 << 6, - /* allocate from non-dma32 zone */ - RKNPU_MEM_NON_DMA32 = 1 << 7, + /* allocate from dma32 zone */ + RKNPU_MEM_DMA32 = 1 << 7, /* request SRAM */ RKNPU_MEM_TRY_ALLOC_SRAM = 1 << 8, + /* request NBUF */ + RKNPU_MEM_TRY_ALLOC_NBUF = 1 << 9, + /* IOMMU limiting IOVA alignment */ + RKNPU_MEM_IOMMU_LIMIT_IOVA_ALIGNMENT = 1 << 10, RKNPU_MEM_MASK = RKNPU_MEM_NON_CONTIGUOUS | RKNPU_MEM_CACHEABLE | RKNPU_MEM_WRITE_COMBINE | RKNPU_MEM_KERNEL_MAPPING | RKNPU_MEM_IOMMU | RKNPU_MEM_ZEROING | - RKNPU_MEM_SECURE | RKNPU_MEM_NON_DMA32 | - RKNPU_MEM_TRY_ALLOC_SRAM + RKNPU_MEM_SECURE | RKNPU_MEM_DMA32 | + RKNPU_MEM_TRY_ALLOC_SRAM | RKNPU_MEM_TRY_ALLOC_NBUF | + RKNPU_MEM_IOMMU_LIMIT_IOVA_ALIGNMENT }; /* sync mode definitions. */ enum e_rknpu_mem_sync_mode { RKNPU_MEM_SYNC_TO_DEVICE = 1 << 0, RKNPU_MEM_SYNC_FROM_DEVICE = 1 << 1, - RKNPU_MEM_SYNC_MASK = - RKNPU_MEM_SYNC_TO_DEVICE | RKNPU_MEM_SYNC_FROM_DEVICE + RKNPU_MEM_SYNC_MASK = RKNPU_MEM_SYNC_TO_DEVICE | + RKNPU_MEM_SYNC_FROM_DEVICE }; /* job mode definitions. */ @@ -132,6 +132,8 @@ enum e_rknpu_action { RKNPU_POWER_OFF = 21, RKNPU_GET_TOTAL_SRAM_SIZE = 22, RKNPU_GET_FREE_SRAM_SIZE = 23, + RKNPU_GET_IOMMU_DOMAIN_ID = 24, + RKNPU_SET_IOMMU_DOMAIN_ID = 25, }; /** @@ -145,6 +147,8 @@ enum e_rknpu_action { * @dma_addr: dma address that access by rknpu. * @sram_size: user-desired sram memory allocation size. * - this size value would be page-aligned internally. + * @iommu_domain_id: iommu domain id + * @reserved: just padding to be 64-bit aligned. */ struct rknpu_mem_create { __u32 handle; @@ -153,6 +157,8 @@ struct rknpu_mem_create { __u64 obj_addr; __u64 dma_addr; __u64 sram_size; + __s32 iommu_domain_id; + __u32 core_mask; }; /** @@ -247,9 +253,10 @@ struct rknpu_subcore_task { * @task_counter: task counter * @priority: submit priority * @task_obj_addr: address of task object - * @regcfg_obj_addr: address of register config object + * @iommu_domain_id: iommu domain id + * @reserved: just padding to be 64-bit aligned. * @task_base_addr: task base address - * @user_data: (optional) user data + * @hw_elapse_time: hardware elapse time * @core_mask: core mask of rknpu * @fence_fd: dma fence fd * @subcore_task: subcore task @@ -263,9 +270,10 @@ struct rknpu_submit { __u32 task_counter; __s32 priority; __u64 task_obj_addr; - __u64 regcfg_obj_addr; + __u32 iommu_domain_id; + __u32 reserved; __u64 task_base_addr; - __u64 user_data; + __s64 hw_elapse_time; __u32 core_mask; __s32 fence_fd; struct rknpu_subcore_task subcore_task[5]; @@ -297,25 +305,25 @@ struct rknpu_action { #include -#define DRM_IOCTL_RKNPU_ACTION \ +#define DRM_IOCTL_RKNPU_ACTION \ DRM_IOWR(DRM_COMMAND_BASE + RKNPU_ACTION, struct rknpu_action) -#define DRM_IOCTL_RKNPU_SUBMIT \ +#define DRM_IOCTL_RKNPU_SUBMIT \ DRM_IOWR(DRM_COMMAND_BASE + RKNPU_SUBMIT, struct rknpu_submit) -#define DRM_IOCTL_RKNPU_MEM_CREATE \ +#define DRM_IOCTL_RKNPU_MEM_CREATE \ DRM_IOWR(DRM_COMMAND_BASE + RKNPU_MEM_CREATE, struct rknpu_mem_create) -#define DRM_IOCTL_RKNPU_MEM_MAP \ +#define DRM_IOCTL_RKNPU_MEM_MAP \ DRM_IOWR(DRM_COMMAND_BASE + RKNPU_MEM_MAP, struct rknpu_mem_map) -#define DRM_IOCTL_RKNPU_MEM_DESTROY \ +#define DRM_IOCTL_RKNPU_MEM_DESTROY \ DRM_IOWR(DRM_COMMAND_BASE + RKNPU_MEM_DESTROY, struct rknpu_mem_destroy) -#define DRM_IOCTL_RKNPU_MEM_SYNC \ +#define DRM_IOCTL_RKNPU_MEM_SYNC \ DRM_IOWR(DRM_COMMAND_BASE + RKNPU_MEM_SYNC, struct rknpu_mem_sync) #define IOCTL_RKNPU_ACTION RKNPU_IOWR(RKNPU_ACTION, struct rknpu_action) #define IOCTL_RKNPU_SUBMIT RKNPU_IOWR(RKNPU_SUBMIT, struct rknpu_submit) -#define IOCTL_RKNPU_MEM_CREATE \ +#define IOCTL_RKNPU_MEM_CREATE \ RKNPU_IOWR(RKNPU_MEM_CREATE, struct rknpu_mem_create) #define IOCTL_RKNPU_MEM_MAP RKNPU_IOWR(RKNPU_MEM_MAP, struct rknpu_mem_map) -#define IOCTL_RKNPU_MEM_DESTROY \ +#define IOCTL_RKNPU_MEM_DESTROY \ RKNPU_IOWR(RKNPU_MEM_DESTROY, struct rknpu_mem_destroy) #define IOCTL_RKNPU_MEM_SYNC RKNPU_IOWR(RKNPU_MEM_SYNC, struct rknpu_mem_sync) diff --git a/target/linux/rockchip/rk35xx/files/drivers/rknpu/include/rknpu_iommu.h b/target/linux/rockchip/rk35xx/files/drivers/rknpu/include/rknpu_iommu.h new file mode 100644 index 0000000000..40ea58e282 --- /dev/null +++ b/target/linux/rockchip/rk35xx/files/drivers/rknpu/include/rknpu_iommu.h @@ -0,0 +1,60 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) Rockchip Electronics Co.Ltd + * Author: Felix Zeng + */ + +#ifndef __LINUX_RKNPU_IOMMU_H +#define __LINUX_RKNPU_IOMMU_H + +#include +#include +#include +#include +#include + +#if KERNEL_VERSION(6, 1, 0) > LINUX_VERSION_CODE +#include +#endif + +#include "rknpu_drv.h" + +enum iommu_dma_cookie_type { + IOMMU_DMA_IOVA_COOKIE, + IOMMU_DMA_MSI_COOKIE, +}; + +struct rknpu_iommu_dma_cookie { + enum iommu_dma_cookie_type type; + + /* Full allocator for IOMMU_DMA_IOVA_COOKIE */ + struct iova_domain iovad; +}; + +dma_addr_t rknpu_iommu_dma_alloc_iova(struct iommu_domain *domain, size_t size, + u64 dma_limit, struct device *dev, + bool size_aligned); + +void rknpu_iommu_dma_free_iova(struct rknpu_iommu_dma_cookie *cookie, + dma_addr_t iova, size_t size, bool size_aligned); + +int rknpu_iommu_dma_map_sg(struct device *dev, struct scatterlist *sg, + int nents, enum dma_data_direction dir, + bool iova_aligned); + +void rknpu_iommu_dma_unmap_sg(struct device *dev, struct scatterlist *sg, + int nents, enum dma_data_direction dir, + bool iova_aligned); + +int rknpu_iommu_init_domain(struct rknpu_device *rknpu_dev); +int rknpu_iommu_switch_domain(struct rknpu_device *rknpu_dev, int domain_id); +void rknpu_iommu_free_domains(struct rknpu_device *rknpu_dev); +int rknpu_iommu_domain_get_and_switch(struct rknpu_device *rknpu_dev, + int domain_id); +int rknpu_iommu_domain_put(struct rknpu_device *rknpu_dev); + +#if KERNEL_VERSION(5, 10, 0) < LINUX_VERSION_CODE +int iommu_get_dma_cookie(struct iommu_domain *domain); +#endif + +#endif diff --git a/target/linux/rockchip/rk35xx/files/drivers/rknpu/include/rknpu_job.h b/target/linux/rockchip/rk35xx/files/drivers/rknpu/include/rknpu_job.h index 6ef52d4392..46eb304ac3 100644 --- a/target/linux/rockchip/rk35xx/files/drivers/rknpu/include/rknpu_job.h +++ b/target/linux/rockchip/rk35xx/files/drivers/rknpu/include/rknpu_job.h @@ -30,7 +30,6 @@ struct rknpu_job { struct rknpu_device *rknpu_dev; struct list_head head[RKNPU_MAX_CORES]; struct work_struct cleanup_work; - bool in_queue[RKNPU_MAX_CORES]; bool irq_entry[RKNPU_MAX_CORES]; unsigned int flags; int ret; @@ -43,9 +42,13 @@ struct rknpu_job { struct dma_fence *fence; ktime_t timestamp; uint32_t use_core_num; - uint32_t run_count; - uint32_t interrupt_count; + atomic_t run_count; + atomic_t interrupt_count; + ktime_t hw_commit_time; ktime_t hw_recoder_time; + ktime_t hw_elapse_time; + atomic_t submit_count[RKNPU_MAX_CORES]; + int iommu_domain_id; }; irqreturn_t rknpu_core0_irq_handler(int irq, void *data); diff --git a/target/linux/rockchip/rk35xx/files/drivers/rknpu/include/rknpu_mem.h b/target/linux/rockchip/rk35xx/files/drivers/rknpu/include/rknpu_mem.h index 925535c85f..9cc5d1d425 100644 --- a/target/linux/rockchip/rk35xx/files/drivers/rknpu/include/rknpu_mem.h +++ b/target/linux/rockchip/rk35xx/files/drivers/rknpu/include/rknpu_mem.h @@ -33,11 +33,14 @@ struct rknpu_mem_object { struct page **pages; struct sg_table *sgt; struct dma_buf *dmabuf; + struct list_head head; unsigned int owner; }; -int rknpu_mem_create_ioctl(struct rknpu_device *rknpu_dev, unsigned long data); -int rknpu_mem_destroy_ioctl(struct rknpu_device *rknpu_dev, unsigned long data); +int rknpu_mem_create_ioctl(struct rknpu_device *rknpu_dev, struct file *file, + unsigned int cmd, unsigned long data); +int rknpu_mem_destroy_ioctl(struct rknpu_device *rknpu_dev, struct file *file, + unsigned long data); int rknpu_mem_sync_ioctl(struct rknpu_device *rknpu_dev, unsigned long data); #endif diff --git a/target/linux/rockchip/rk35xx/files/drivers/rknpu/include/rknpu_mm.h b/target/linux/rockchip/rk35xx/files/drivers/rknpu/include/rknpu_mm.h index b764892d1d..73ae6d7cef 100644 --- a/target/linux/rockchip/rk35xx/files/drivers/rknpu/include/rknpu_mm.h +++ b/target/linux/rockchip/rk35xx/files/drivers/rknpu/include/rknpu_mm.h @@ -10,7 +10,6 @@ #include #include #include -#include #include #include "rknpu_drv.h" @@ -40,22 +39,4 @@ int rknpu_mm_free(struct rknpu_mm *mm, struct rknpu_mm_obj *mm_obj); int rknpu_mm_dump(struct seq_file *m, void *data); -enum iommu_dma_cookie_type { - IOMMU_DMA_IOVA_COOKIE, - IOMMU_DMA_MSI_COOKIE, -}; - -struct rknpu_iommu_dma_cookie { - enum iommu_dma_cookie_type type; - - /* Full allocator for IOMMU_DMA_IOVA_COOKIE */ - struct iova_domain iovad; -}; - -dma_addr_t rknpu_iommu_dma_alloc_iova(struct iommu_domain *domain, size_t size, - u64 dma_limit, struct device *dev); - -void rknpu_iommu_dma_free_iova(struct rknpu_iommu_dma_cookie *cookie, - dma_addr_t iova, size_t size); - #endif diff --git a/target/linux/rockchip/rk35xx/files/drivers/rknpu/rknpu_debugger.c b/target/linux/rockchip/rk35xx/files/drivers/rknpu/rknpu_debugger.c index 146aa7dfc8..0cfec7fe7c 100644 --- a/target/linux/rockchip/rk35xx/files/drivers/rknpu/rknpu_debugger.c +++ b/target/linux/rockchip/rk35xx/files/drivers/rknpu/rknpu_debugger.c @@ -46,7 +46,7 @@ static int rknpu_load_show(struct seq_file *m, void *data) unsigned long flags; int i; int load; - uint64_t busy_time_total, div_value; + uint64_t total_busy_time, div_value; seq_puts(m, "NPU load: "); for (i = 0; i < rknpu_dev->config->num_irqs; i++) { @@ -57,13 +57,13 @@ static int rknpu_load_show(struct seq_file *m, void *data) spin_lock_irqsave(&rknpu_dev->irq_lock, flags); - busy_time_total = subcore_data->timer.busy_time_record; + total_busy_time = subcore_data->timer.total_busy_time; spin_unlock_irqrestore(&rknpu_dev->irq_lock, flags); - div_value = (RKNPU_LOAD_INTERVAL / 100000); - do_div(busy_time_total, div_value); - load = busy_time_total; + div_value = (RKNPU_LOAD_INTERVAL / 100); + do_div(total_busy_time, div_value); + load = total_busy_time > 100 ? 100 : total_busy_time; if (rknpu_dev->config->num_irqs > 1) seq_printf(m, "%2.d%%,", load); @@ -457,7 +457,11 @@ CREATE_FAIL: #ifdef CONFIG_ROCKCHIP_RKNPU_PROC_FS static int rknpu_procfs_open(struct inode *inode, struct file *file) { +#if KERNEL_VERSION(6, 1, 0) > LINUX_VERSION_CODE struct rknpu_debugger_node *node = PDE_DATA(inode); +#else + struct rknpu_debugger_node *node = pde_data(inode); +#endif return single_open(file, node->info_ent->show, node); } diff --git a/target/linux/rockchip/rk35xx/files/drivers/rknpu/rknpu_devfreq.c b/target/linux/rockchip/rk35xx/files/drivers/rknpu/rknpu_devfreq.c new file mode 100644 index 0000000000..2668d2c466 --- /dev/null +++ b/target/linux/rockchip/rk35xx/files/drivers/rknpu/rknpu_devfreq.c @@ -0,0 +1,807 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) Rockchip Electronics Co.Ltd + * Author: Finley Xiao + */ + +#include +#include +#include +#include +#include +#include <../drivers/devfreq/governor.h> +#include "rknpu_drv.h" +#include "rknpu_devfreq.h" + +#define POWER_DOWN_FREQ 200000000 + +static int npu_devfreq_target(struct device *dev, unsigned long *freq, + u32 flags); + +static struct monitor_dev_profile npu_mdevp = { + .type = MONITOR_TYPE_DEV, + .low_temp_adjust = rockchip_monitor_dev_low_temp_adjust, + .high_temp_adjust = rockchip_monitor_dev_high_temp_adjust, +#if KERNEL_VERSION(6, 1, 0) <= LINUX_VERSION_CODE + .check_rate_volt = rockchip_monitor_check_rate_volt, +#else + .update_volt = rockchip_monitor_check_rate_volt, +#endif +}; + +static int npu_devfreq_get_dev_status(struct device *dev, + struct devfreq_dev_status *stat) +{ + return 0; +} + +static int npu_devfreq_get_cur_freq(struct device *dev, unsigned long *freq) +{ + struct rknpu_device *rknpu_dev = dev_get_drvdata(dev); + + *freq = rknpu_dev->current_freq; + + return 0; +} + +static struct devfreq_dev_profile npu_devfreq_profile = { + .polling_ms = 50, + .target = npu_devfreq_target, + .get_dev_status = npu_devfreq_get_dev_status, + .get_cur_freq = npu_devfreq_get_cur_freq, +}; + +static int devfreq_rknpu_ondemand_func(struct devfreq *df, unsigned long *freq) +{ + struct rknpu_device *rknpu_dev = df->data; + + if (rknpu_dev && rknpu_dev->ondemand_freq) + *freq = rknpu_dev->ondemand_freq; + else + *freq = df->previous_freq; + + return 0; +} + +static int devfreq_rknpu_ondemand_handler(struct devfreq *devfreq, + unsigned int event, void *data) +{ + return 0; +} + +static struct devfreq_governor devfreq_rknpu_ondemand = { + .name = "rknpu_ondemand", + .get_target_freq = devfreq_rknpu_ondemand_func, + .event_handler = devfreq_rknpu_ondemand_handler, +}; + +static int rk3576_npu_set_read_margin(struct device *dev, + struct rockchip_opp_info *opp_info, + u32 rm) +{ + if (!opp_info->grf || !opp_info->volt_rm_tbl) + return 0; + + if (rm == opp_info->current_rm || rm == UINT_MAX) + return 0; + + LOG_DEV_DEBUG(dev, "set rm to %d\n", rm); + + regmap_write(opp_info->grf, 0x08, 0x001c0000 | (rm << 2)); + regmap_write(opp_info->grf, 0x0c, 0x003c0000 | (rm << 2)); + regmap_write(opp_info->grf, 0x10, 0x001c0000 | (rm << 2)); + + return 0; +} + +static int rk3588_npu_get_soc_info(struct device *dev, struct device_node *np, + int *bin, int *process) +{ + int ret = 0; + u8 value = 0; + + if (!bin) + return 0; + + if (of_property_match_string(np, "nvmem-cell-names", + "specification_serial_number") >= 0) { + ret = rockchip_nvmem_cell_read_u8( + np, "specification_serial_number", &value); + if (ret) { + LOG_DEV_ERROR( + dev, + "Failed to get specification_serial_number\n"); + return ret; + } + /* RK3588M */ + if (value == 0xd) + *bin = 1; + /* RK3588J */ + else if (value == 0xa) + *bin = 2; + } + if (*bin < 0) + *bin = 0; + LOG_DEV_INFO(dev, "bin=%d\n", *bin); + + return ret; +} + +#if KERNEL_VERSION(6, 1, 0) <= LINUX_VERSION_CODE +static int rk3588_npu_set_soc_info(struct device *dev, struct device_node *np, + struct rockchip_opp_info *opp_info) +{ + int bin = opp_info->bin; + + if (opp_info->volt_sel < 0) + return 0; + if (bin < 0) + bin = 0; + + if (!of_property_read_bool(np, "rockchip,supported-hw")) + return 0; + + /* SoC Version */ + opp_info->supported_hw[0] = BIT(bin); + /* Speed Grade */ + opp_info->supported_hw[1] = BIT(opp_info->volt_sel); + + return 0; +} +#else +static int rk3588_npu_set_soc_info(struct device *dev, struct device_node *np, + int bin, int process, int volt_sel) +{ + struct opp_table *opp_table; + u32 supported_hw[2]; + + if (volt_sel < 0) + return 0; + if (bin < 0) + bin = 0; + + if (!of_property_read_bool(np, "rockchip,supported-hw")) + return 0; + + /* SoC Version */ + supported_hw[0] = BIT(bin); + /* Speed Grade */ + supported_hw[1] = BIT(volt_sel); + opp_table = dev_pm_opp_set_supported_hw(dev, supported_hw, 2); + if (IS_ERR(opp_table)) { + LOG_DEV_ERROR(dev, "failed to set supported opp\n"); + return PTR_ERR(opp_table); + } + + return 0; +} +#endif + +static int rk3588_npu_set_read_margin(struct device *dev, + struct rockchip_opp_info *opp_info, + u32 rm) +{ + u32 offset = 0, val = 0; + int i, ret = 0; + + if (!opp_info->grf || !opp_info->volt_rm_tbl) + return 0; + + if (rm == opp_info->current_rm || rm == UINT_MAX) + return 0; + + LOG_DEV_DEBUG(dev, "set rm to %d\n", rm); + + for (i = 0; i < 3; i++) { + ret = regmap_read(opp_info->grf, offset, &val); + if (ret < 0) { + LOG_DEV_ERROR(dev, "failed to get rm from 0x%x\n", + offset); + return ret; + } + val &= ~0x1c; + regmap_write(opp_info->grf, offset, val | (rm << 2)); + offset += 4; + } + opp_info->current_rm = rm; + + return 0; +} + +#if KERNEL_VERSION(6, 1, 0) <= LINUX_VERSION_CODE +static int npu_opp_config_regulators(struct device *dev, + struct dev_pm_opp *old_opp, + struct dev_pm_opp *new_opp, + struct regulator **regulators, + unsigned int count) +{ + struct rknpu_device *rknpu_dev = dev_get_drvdata(dev); + + return rockchip_opp_config_regulators(dev, old_opp, new_opp, regulators, + count, &rknpu_dev->opp_info); +} + +static int npu_opp_config_clks(struct device *dev, struct opp_table *opp_table, + struct dev_pm_opp *opp, void *data, + bool scaling_down) +{ + struct rknpu_device *rknpu_dev = dev_get_drvdata(dev); + + return rockchip_opp_config_clks(dev, opp_table, opp, data, scaling_down, + &rknpu_dev->opp_info); +} +#endif + +static const struct rockchip_opp_data rk3576_npu_opp_data = { + .set_read_margin = rk3576_npu_set_read_margin, +#if KERNEL_VERSION(6, 1, 0) <= LINUX_VERSION_CODE + .config_regulators = npu_opp_config_regulators, + .config_clks = npu_opp_config_clks, +#endif +}; + +static const struct rockchip_opp_data rk3588_npu_opp_data = { + .get_soc_info = rk3588_npu_get_soc_info, + .set_soc_info = rk3588_npu_set_soc_info, + .set_read_margin = rk3588_npu_set_read_margin, +#if KERNEL_VERSION(6, 1, 0) <= LINUX_VERSION_CODE + .config_regulators = npu_opp_config_regulators, + .config_clks = npu_opp_config_clks, +#endif +}; + +static const struct of_device_id rockchip_npu_of_match[] = { + { + .compatible = "rockchip,rk3576", + .data = (void *)&rk3576_npu_opp_data, + }, + { + .compatible = "rockchip,rk3588", + .data = (void *)&rk3588_npu_opp_data, + }, + {}, +}; + +#if KERNEL_VERSION(6, 1, 0) <= LINUX_VERSION_CODE +void rknpu_devfreq_lock(struct rknpu_device *rknpu_dev) +{ + if (rknpu_dev->devfreq) + rockchip_opp_dvfs_lock(&rknpu_dev->opp_info); +} +EXPORT_SYMBOL(rknpu_devfreq_lock); + +void rknpu_devfreq_unlock(struct rknpu_device *rknpu_dev) +{ + if (rknpu_dev->devfreq) + rockchip_opp_dvfs_unlock(&rknpu_dev->opp_info); +} +EXPORT_SYMBOL(rknpu_devfreq_unlock); + +static int npu_devfreq_target(struct device *dev, unsigned long *freq, + u32 flags) +{ + struct rknpu_device *rknpu_dev = dev_get_drvdata(dev); + struct rockchip_opp_info *opp_info = &rknpu_dev->opp_info; + struct dev_pm_opp *opp; + unsigned long opp_volt; + int ret = 0; + + if (!opp_info->is_rate_volt_checked) + return -EINVAL; + + opp = devfreq_recommended_opp(dev, freq, flags); + if (IS_ERR(opp)) + return PTR_ERR(opp); + opp_volt = dev_pm_opp_get_voltage(opp); + dev_pm_opp_put(opp); + + if (*freq == rknpu_dev->current_freq) + return 0; + + rockchip_opp_dvfs_lock(opp_info); + if (pm_runtime_active(dev)) + opp_info->is_runtime_active = true; + else + opp_info->is_runtime_active = false; + ret = dev_pm_opp_set_rate(dev, *freq); + if (!ret) { + rknpu_dev->current_freq = *freq; + if (rknpu_dev->devfreq) + rknpu_dev->devfreq->last_status.current_frequency = + *freq; + rknpu_dev->current_volt = opp_volt; + LOG_DEV_DEBUG(dev, "set rknpu freq: %lu, volt: %lu\n", + rknpu_dev->current_freq, rknpu_dev->current_volt); + } + rockchip_opp_dvfs_unlock(opp_info); + + return ret; +} + +static const struct rockchip_opp_data rockchip_npu_opp_data = { + .config_clks = npu_opp_config_clks, +}; + +int rknpu_devfreq_init(struct rknpu_device *rknpu_dev) +{ + struct rockchip_opp_info *info = &rknpu_dev->opp_info; + struct device *dev = rknpu_dev->dev; + struct devfreq_dev_profile *dp; + struct dev_pm_opp *opp; + unsigned int dyn_power_coeff = 0; + int ret = 0; + + info->data = &rockchip_npu_opp_data; + rockchip_get_opp_data(rockchip_npu_of_match, info); + ret = rockchip_init_opp_table(dev, info, "clk_npu", "rknpu"); + if (ret) { + LOG_DEV_ERROR(dev, "failed to init_opp_table\n"); + return -EINVAL; + } + + rknpu_dev->current_freq = clk_get_rate(rknpu_dev->clks[0].clk); + opp = devfreq_recommended_opp(dev, &rknpu_dev->current_freq, 0); + if (IS_ERR(opp)) { + ret = PTR_ERR(opp); + goto err_uinit_table; + } + dev_pm_opp_put(opp); + + dp = &npu_devfreq_profile; + dp->initial_freq = rknpu_dev->current_freq; + of_property_read_u32(dev->of_node, "dynamic-power-coefficient", + &dyn_power_coeff); + if (dyn_power_coeff) + dp->is_cooling_device = true; + + ret = devfreq_add_governor(&devfreq_rknpu_ondemand); + if (ret) { + LOG_DEV_ERROR(dev, "failed to add rknpu_ondemand governor\n"); + goto err_uinit_table; + } + + rknpu_dev->devfreq = devm_devfreq_add_device(dev, dp, "rknpu_ondemand", + (void *)rknpu_dev); + if (IS_ERR(rknpu_dev->devfreq)) { + LOG_DEV_ERROR(dev, "failed to add devfreq\n"); + ret = PTR_ERR(rknpu_dev->devfreq); + rknpu_dev->devfreq = NULL; + goto err_remove_governor; + } + + npu_mdevp.data = rknpu_dev->devfreq; + npu_mdevp.opp_info = &rknpu_dev->opp_info; + rknpu_dev->mdev_info = + rockchip_system_monitor_register(dev, &npu_mdevp); + if (IS_ERR(rknpu_dev->mdev_info)) { + dev_dbg(dev, "without system monitor\n"); + rknpu_dev->mdev_info = NULL; + } + + rknpu_dev->current_freq = clk_get_rate(rknpu_dev->clks[0].clk); + rknpu_dev->ondemand_freq = rknpu_dev->current_freq; + rknpu_dev->current_volt = regulator_get_voltage(rknpu_dev->vdd); + + rknpu_dev->devfreq->previous_freq = rknpu_dev->current_freq; + if (rknpu_dev->devfreq->suspend_freq) + rknpu_dev->devfreq->resume_freq = rknpu_dev->current_freq; + rknpu_dev->devfreq->last_status.current_frequency = + rknpu_dev->current_freq; + rknpu_dev->devfreq->last_status.total_time = 1; + rknpu_dev->devfreq->last_status.busy_time = 1; + + return 0; + +err_remove_governor: + devfreq_remove_governor(&devfreq_rknpu_ondemand); +err_uinit_table: +#if KERNEL_VERSION(5, 10, 198) <= LINUX_VERSION_CODE + rockchip_uninit_opp_table(dev, info); +#else + dev_pm_opp_of_remove_table(dev); +#endif + + return ret; +} +EXPORT_SYMBOL(rknpu_devfreq_init); + +int rknpu_devfreq_runtime_suspend(struct device *dev) +{ + struct rknpu_device *rknpu_dev = dev_get_drvdata(dev); + struct rockchip_opp_info *opp_info = &rknpu_dev->opp_info; + + if (opp_info->is_scmi_clk) { + if (clk_set_rate(opp_info->clk, POWER_DOWN_FREQ)) + LOG_DEV_ERROR(dev, "failed to restore clk rate\n"); + } + opp_info->current_rm = UINT_MAX; + + return 0; +} +EXPORT_SYMBOL(rknpu_devfreq_runtime_suspend); + +int rknpu_devfreq_runtime_resume(struct device *dev) +{ + struct rknpu_device *rknpu_dev = dev_get_drvdata(dev); + struct rockchip_opp_info *opp_info = &rknpu_dev->opp_info; + int ret = 0; + + if (!rknpu_dev->current_freq || !rknpu_dev->current_volt) + return 0; + + ret = clk_bulk_prepare_enable(opp_info->nclocks, opp_info->clocks); + if (ret) { + LOG_DEV_INFO(dev, "failed to enable opp clks\n"); + return ret; + } + + if (opp_info->data && opp_info->data->set_read_margin) + opp_info->data->set_read_margin(dev, opp_info, + opp_info->target_rm); + if (opp_info->is_scmi_clk) { + if (clk_set_rate(opp_info->clk, rknpu_dev->current_freq)) + LOG_DEV_ERROR(dev, "failed to set power down rate\n"); + } + + clk_bulk_disable_unprepare(opp_info->nclocks, opp_info->clocks); + + return ret; +} +EXPORT_SYMBOL(rknpu_devfreq_runtime_resume); +#else +void rknpu_devfreq_lock(struct rknpu_device *rknpu_dev) +{ + rockchip_monitor_volt_adjust_lock(rknpu_dev->mdev_info); +} +EXPORT_SYMBOL(rknpu_devfreq_lock); + +void rknpu_devfreq_unlock(struct rknpu_device *rknpu_dev) +{ + rockchip_monitor_volt_adjust_unlock(rknpu_dev->mdev_info); +} +EXPORT_SYMBOL(rknpu_devfreq_unlock); + +static int npu_opp_helper(struct dev_pm_set_opp_data *data) +{ + struct device *dev = data->dev; + struct dev_pm_opp_supply *old_supply_vdd = &data->old_opp.supplies[0]; + struct dev_pm_opp_supply *old_supply_mem = &data->old_opp.supplies[1]; + struct dev_pm_opp_supply *new_supply_vdd = &data->new_opp.supplies[0]; + struct dev_pm_opp_supply *new_supply_mem = &data->new_opp.supplies[1]; + struct regulator *vdd_reg = data->regulators[0]; + struct regulator *mem_reg = data->regulators[1]; + struct clk *clk = data->clk; + struct rknpu_device *rknpu_dev = dev_get_drvdata(dev); + struct rockchip_opp_info *opp_info = &rknpu_dev->opp_info; + unsigned long old_freq = data->old_opp.rate; + unsigned long new_freq = data->new_opp.rate; + bool is_set_rm = true; + bool is_set_clk = true; + u32 target_rm = UINT_MAX; + int ret = 0; + + if (!pm_runtime_active(dev)) { + is_set_rm = false; + if (opp_info->scmi_clk) + is_set_clk = false; + } + + ret = clk_bulk_prepare_enable(opp_info->num_clks, opp_info->clks); + if (ret < 0) { + LOG_DEV_ERROR(dev, "failed to enable opp clks\n"); + return ret; + } + rockchip_get_read_margin(dev, opp_info, new_supply_vdd->u_volt, + &target_rm); + + /* Change frequency */ + LOG_DEV_DEBUG(dev, "switching OPP: %lu Hz --> %lu Hz\n", old_freq, + new_freq); + /* Scaling up? Scale voltage before frequency */ + if (new_freq >= old_freq) { + rockchip_set_intermediate_rate(dev, opp_info, clk, old_freq, + new_freq, true, is_set_clk); + ret = regulator_set_voltage(mem_reg, new_supply_mem->u_volt, + INT_MAX); + if (ret) { + LOG_DEV_ERROR(dev, + "failed to set volt %lu uV for mem reg\n", + new_supply_mem->u_volt); + goto restore_voltage; + } + ret = regulator_set_voltage(vdd_reg, new_supply_vdd->u_volt, + INT_MAX); + if (ret) { + LOG_DEV_ERROR(dev, + "failed to set volt %lu uV for vdd reg\n", + new_supply_vdd->u_volt); + goto restore_voltage; + } + rockchip_set_read_margin(dev, opp_info, target_rm, is_set_rm); + if (is_set_clk && clk_set_rate(clk, new_freq)) { + ret = -EINVAL; + LOG_DEV_ERROR(dev, "failed to set clk rate: %d\n", ret); + goto restore_rm; + } + /* Scaling down? Scale voltage after frequency */ + } else { + rockchip_set_intermediate_rate(dev, opp_info, clk, old_freq, + new_freq, false, is_set_clk); + rockchip_set_read_margin(dev, opp_info, target_rm, is_set_rm); + if (is_set_clk && clk_set_rate(clk, new_freq)) { + ret = -EINVAL; + LOG_DEV_ERROR(dev, "failed to set clk rate: %d\n", ret); + goto restore_rm; + } + ret = regulator_set_voltage(vdd_reg, new_supply_vdd->u_volt, + INT_MAX); + if (ret) { + LOG_DEV_ERROR(dev, + "failed to set volt %lu uV for vdd reg\n", + new_supply_vdd->u_volt); + goto restore_freq; + } + ret = regulator_set_voltage(mem_reg, new_supply_mem->u_volt, + INT_MAX); + if (ret) { + LOG_DEV_ERROR(dev, + "failed to set volt %lu uV for mem reg\n", + new_supply_mem->u_volt); + goto restore_freq; + } + } + + clk_bulk_disable_unprepare(opp_info->num_clks, opp_info->clks); + + return 0; + +restore_freq: + if (is_set_clk && clk_set_rate(clk, old_freq)) + LOG_DEV_ERROR(dev, "failed to restore old-freq %lu Hz\n", + old_freq); +restore_rm: + rockchip_get_read_margin(dev, opp_info, old_supply_vdd->u_volt, + &target_rm); + rockchip_set_read_margin(dev, opp_info, opp_info->current_rm, + is_set_rm); +restore_voltage: + regulator_set_voltage(mem_reg, old_supply_mem->u_volt, INT_MAX); + regulator_set_voltage(vdd_reg, old_supply_vdd->u_volt, INT_MAX); + clk_bulk_disable_unprepare(opp_info->num_clks, opp_info->clks); + + return ret; +} + +static int npu_devfreq_target(struct device *dev, unsigned long *freq, + u32 flags) +{ + struct rknpu_device *rknpu_dev = dev_get_drvdata(dev); + struct dev_pm_opp *opp; + unsigned long opp_volt; + int ret = 0; + + if (!npu_mdevp.is_checked) + return -EINVAL; + + opp = devfreq_recommended_opp(dev, freq, flags); + if (IS_ERR(opp)) + return PTR_ERR(opp); + opp_volt = dev_pm_opp_get_voltage(opp); + dev_pm_opp_put(opp); + + rockchip_monitor_volt_adjust_lock(rknpu_dev->mdev_info); + + ret = dev_pm_opp_set_rate(dev, *freq); + if (!ret) { + rknpu_dev->current_freq = *freq; + if (rknpu_dev->devfreq) + rknpu_dev->devfreq->last_status.current_frequency = + *freq; + rknpu_dev->current_volt = opp_volt; + LOG_DEV_DEBUG(dev, "set rknpu freq: %lu, volt: %lu\n", + rknpu_dev->current_freq, rknpu_dev->current_volt); + } + + rockchip_monitor_volt_adjust_unlock(rknpu_dev->mdev_info); + + return ret; +} + +static unsigned long npu_get_static_power(struct devfreq *devfreq, + unsigned long voltage) +{ + struct device *dev = devfreq->dev.parent; + struct rknpu_device *rknpu_dev = dev_get_drvdata(dev); + + if (!rknpu_dev->model_data) + return 0; + + return rockchip_ipa_get_static_power(rknpu_dev->model_data, voltage); +} + +static struct devfreq_cooling_power npu_cooling_power = { + .get_static_power = &npu_get_static_power, +}; + +int rknpu_devfreq_init(struct rknpu_device *rknpu_dev) +{ + struct device *dev = rknpu_dev->dev; + struct devfreq_dev_profile *dp = &npu_devfreq_profile; + struct dev_pm_opp *opp; + struct opp_table *reg_table = NULL; + struct opp_table *opp_table = NULL; + const char *const reg_names[] = { "rknpu", "mem" }; + int ret = -EINVAL; + + if (strstr(__clk_get_name(rknpu_dev->clks[0].clk), "scmi")) + rknpu_dev->opp_info.scmi_clk = rknpu_dev->clks[0].clk; + + if (of_find_property(dev->of_node, "rknpu-supply", NULL) && + of_find_property(dev->of_node, "mem-supply", NULL)) { + reg_table = dev_pm_opp_set_regulators(dev, reg_names, 2); + if (IS_ERR(reg_table)) + return PTR_ERR(reg_table); + opp_table = + dev_pm_opp_register_set_opp_helper(dev, npu_opp_helper); + if (IS_ERR(opp_table)) { + dev_pm_opp_put_regulators(reg_table); + return PTR_ERR(opp_table); + } + } else { + reg_table = dev_pm_opp_set_regulators(dev, reg_names, 1); + if (IS_ERR(reg_table)) + return PTR_ERR(reg_table); + } + + rockchip_get_opp_data(rockchip_npu_of_match, &rknpu_dev->opp_info); + ret = rockchip_init_opp_table(dev, &rknpu_dev->opp_info, "npu_leakage", + "rknpu"); + if (ret) { + LOG_DEV_ERROR(dev, "failed to init_opp_table\n"); + return ret; + } + + rknpu_dev->current_freq = clk_get_rate(rknpu_dev->clks[0].clk); + + opp = devfreq_recommended_opp(dev, &rknpu_dev->current_freq, 0); + if (IS_ERR(opp)) { + ret = PTR_ERR(opp); + goto err_remove_table; + } + dev_pm_opp_put(opp); + dp->initial_freq = rknpu_dev->current_freq; + + ret = devfreq_add_governor(&devfreq_rknpu_ondemand); + if (ret) { + LOG_DEV_ERROR(dev, "failed to add rknpu_ondemand governor\n"); + goto err_remove_table; + } + + rknpu_dev->devfreq = devm_devfreq_add_device(dev, dp, "rknpu_ondemand", + (void *)rknpu_dev); + if (IS_ERR(rknpu_dev->devfreq)) { + LOG_DEV_ERROR(dev, "failed to add devfreq\n"); + ret = PTR_ERR(rknpu_dev->devfreq); + goto err_remove_governor; + } + + npu_mdevp.data = rknpu_dev->devfreq; + npu_mdevp.opp_info = &rknpu_dev->opp_info; + rknpu_dev->mdev_info = + rockchip_system_monitor_register(dev, &npu_mdevp); + if (IS_ERR(rknpu_dev->mdev_info)) { + LOG_DEV_DEBUG(dev, "without system monitor\n"); + rknpu_dev->mdev_info = NULL; + npu_mdevp.is_checked = true; + } + rknpu_dev->current_freq = clk_get_rate(rknpu_dev->clks[0].clk); + rknpu_dev->ondemand_freq = rknpu_dev->current_freq; + rknpu_dev->current_volt = regulator_get_voltage(rknpu_dev->vdd); + + rknpu_dev->devfreq->previous_freq = rknpu_dev->current_freq; + if (rknpu_dev->devfreq->suspend_freq) + rknpu_dev->devfreq->resume_freq = rknpu_dev->current_freq; + rknpu_dev->devfreq->last_status.current_frequency = + rknpu_dev->current_freq; + rknpu_dev->devfreq->last_status.total_time = 1; + rknpu_dev->devfreq->last_status.busy_time = 1; + + of_property_read_u32(dev->of_node, "dynamic-power-coefficient", + (u32 *)&npu_cooling_power.dyn_power_coeff); + rknpu_dev->model_data = + rockchip_ipa_power_model_init(dev, "npu_leakage"); + if (IS_ERR_OR_NULL(rknpu_dev->model_data)) { + rknpu_dev->model_data = NULL; + LOG_DEV_ERROR(dev, "failed to initialize power model\n"); + } else if (rknpu_dev->model_data->dynamic_coefficient) { + npu_cooling_power.dyn_power_coeff = + rknpu_dev->model_data->dynamic_coefficient; + } + if (!npu_cooling_power.dyn_power_coeff) { + LOG_DEV_ERROR(dev, "failed to get dynamic-coefficient\n"); + goto out; + } + + rknpu_dev->devfreq_cooling = of_devfreq_cooling_register_power( + dev->of_node, rknpu_dev->devfreq, &npu_cooling_power); + if (IS_ERR_OR_NULL(rknpu_dev->devfreq_cooling)) + LOG_DEV_ERROR(dev, "failed to register cooling device\n"); + +out: + return 0; + +err_remove_governor: + devfreq_remove_governor(&devfreq_rknpu_ondemand); +err_remove_table: +#if KERNEL_VERSION(5, 10, 198) <= LINUX_VERSION_CODE + rockchip_uninit_opp_table(dev, &rknpu_dev->opp_info); +#else + dev_pm_opp_of_remove_table(dev); +#endif + + rknpu_dev->devfreq = NULL; + + return ret; +} +EXPORT_SYMBOL(rknpu_devfreq_init); + +int rknpu_devfreq_runtime_suspend(struct device *dev) +{ + struct rknpu_device *rknpu_dev = dev_get_drvdata(dev); + struct rockchip_opp_info *opp_info = &rknpu_dev->opp_info; + + if (opp_info->scmi_clk) { + if (clk_set_rate(opp_info->scmi_clk, POWER_DOWN_FREQ)) + LOG_DEV_ERROR(dev, "failed to restore clk rate\n"); + } + opp_info->current_rm = UINT_MAX; + + return 0; +} +EXPORT_SYMBOL(rknpu_devfreq_runtime_suspend); + +int rknpu_devfreq_runtime_resume(struct device *dev) +{ + struct rknpu_device *rknpu_dev = dev_get_drvdata(dev); + struct rockchip_opp_info *opp_info = &rknpu_dev->opp_info; + int ret = 0; + + if (!rknpu_dev->current_freq || !rknpu_dev->current_volt) + return 0; + + ret = clk_bulk_prepare_enable(opp_info->num_clks, opp_info->clks); + if (ret) { + LOG_DEV_INFO(dev, "failed to enable opp clks\n"); + return ret; + } + + if (opp_info->data && opp_info->data->set_read_margin) + opp_info->data->set_read_margin(dev, opp_info, + opp_info->target_rm); + if (opp_info->scmi_clk) { + if (clk_set_rate(opp_info->scmi_clk, rknpu_dev->current_freq)) + LOG_DEV_ERROR(dev, "failed to set power down rate\n"); + } + + clk_bulk_disable_unprepare(opp_info->num_clks, opp_info->clks); + + return ret; +} +EXPORT_SYMBOL(rknpu_devfreq_runtime_resume); +#endif + +void rknpu_devfreq_remove(struct rknpu_device *rknpu_dev) +{ + if (rknpu_dev->mdev_info) { + rockchip_system_monitor_unregister(rknpu_dev->mdev_info); + rknpu_dev->mdev_info = NULL; + } + if (rknpu_dev->devfreq) + devfreq_remove_governor(&devfreq_rknpu_ondemand); +#if KERNEL_VERSION(5, 10, 198) <= LINUX_VERSION_CODE + rockchip_uninit_opp_table(rknpu_dev->dev, &rknpu_dev->opp_info); +#else + dev_pm_opp_of_remove_table(rknpu_dev->dev); +#endif +} +EXPORT_SYMBOL(rknpu_devfreq_remove); diff --git a/target/linux/rockchip/rk35xx/files/drivers/rknpu/rknpu_drv.c b/target/linux/rockchip/rk35xx/files/drivers/rknpu/rknpu_drv.c index bbcf19af87..e8afa9778c 100644 --- a/target/linux/rockchip/rk35xx/files/drivers/rknpu/rknpu_drv.c +++ b/target/linux/rockchip/rk35xx/files/drivers/rknpu/rknpu_drv.c @@ -8,7 +8,6 @@ #include #include #include -#include #include #include #include @@ -31,19 +30,11 @@ #include #include #include -#include #include -#include #include #ifndef FPGA_PLATFORM #include -#include -#include -#include -#ifdef CONFIG_PM_DEVFREQ -#include <../drivers/devfreq/governor.h> -#endif #endif #include "rknpu_ioctl.h" @@ -51,6 +42,8 @@ #include "rknpu_fence.h" #include "rknpu_drv.h" #include "rknpu_gem.h" +#include "rknpu_devfreq.h" +#include "rknpu_iommu.h" #ifdef CONFIG_ROCKCHIP_RKNPU_DRM_GEM #include @@ -79,30 +72,93 @@ module_param(bypass_soft_reset, int, 0644); MODULE_PARM_DESC(bypass_soft_reset, "bypass RKNPU soft reset if set it to 1, disabled by default"); -struct rknpu_irqs_data { - const char *name; - irqreturn_t (*irq_hdl)(int irq, void *ctx); -}; - static const struct rknpu_irqs_data rknpu_irqs[] = { { "npu_irq", rknpu_core0_irq_handler } }; +static const struct rknpu_irqs_data rk3576_npu_irqs[] = { + { "npu0_irq", rknpu_core0_irq_handler }, + { "npu1_irq", rknpu_core1_irq_handler } +}; + static const struct rknpu_irqs_data rk3588_npu_irqs[] = { { "npu0_irq", rknpu_core0_irq_handler }, { "npu1_irq", rknpu_core1_irq_handler }, { "npu2_irq", rknpu_core2_irq_handler } }; -static const struct rknpu_reset_data rknpu_resets[] = { { "srst_a", - "srst_h" } }; - -static const struct rknpu_reset_data rk3588_npu_resets[] = { - { "srst_a0", "srst_h0" }, - { "srst_a1", "srst_h1" }, - { "srst_a2", "srst_h2" } +static const struct rknpu_amount_data rknpu_old_top_amount = { + .offset_clr_all = 0x8010, + .offset_dt_wr = 0x8034, + .offset_dt_rd = 0x8038, + .offset_wt_rd = 0x803c, }; +static const struct rknpu_amount_data rknpu_top_amount = { + .offset_clr_all = 0x2210, + .offset_dt_wr = 0x2234, + .offset_dt_rd = 0x2238, + .offset_wt_rd = 0x223c +}; + +static const struct rknpu_amount_data rknpu_core_amount = { + .offset_clr_all = 0x2410, + .offset_dt_wr = 0x2434, + .offset_dt_rd = 0x2438, + .offset_wt_rd = 0x243c, +}; + +static void rk3576_state_init(struct rknpu_device *rknpu_dev) +{ + void __iomem *rknpu_core_base = rknpu_dev->base[0]; + + writel(0x1, rknpu_core_base + 0x10); + writel(0, rknpu_core_base + 0x1004); + writel(0x80000000, rknpu_core_base + 0x1024); + writel(1, rknpu_core_base + 0x1004); + writel(0x80000000, rknpu_core_base + 0x1024); + writel(0x1e, rknpu_core_base + 0x1004); +} + +static int rk3576_cache_sgt_init(struct rknpu_device *rknpu_dev) +{ + struct sg_table *sgt = NULL; + struct scatterlist *sgl = NULL; + uint64_t block_size_kb[4] = { 448, 64, 448, 64 }; + uint64_t block_offset_kb[4] = { 0, 896, 448, 960 }; + int core_num = rknpu_dev->config->num_irqs; + int ret = 0, i = 0, j = 0; + + for (i = 0; i < core_num; i++) { + sgt = kzalloc(sizeof(struct sg_table), GFP_KERNEL); + if (!sgt) + goto out_free_table; + ret = sg_alloc_table(sgt, core_num, GFP_KERNEL); + if (ret) { + kfree(sgt); + goto out_free_table; + } + rknpu_dev->cache_sgt[i] = sgt; + for_each_sgtable_sg(sgt, sgl, j) { + sg_set_page(sgl, NULL, + block_size_kb[i * core_num + j] * 1024, + block_offset_kb[i * core_num + j] * 1024); + } + } + return 0; + +out_free_table: + for (i = 0; i < core_num; i++) { + if (rknpu_dev->cache_sgt[i]) { + sg_free_table(rknpu_dev->cache_sgt[i]); + kfree(rknpu_dev->cache_sgt[i]); + rknpu_dev->cache_sgt[i] = NULL; + } + } + + return ret; +} + static const struct rknpu_config rk356x_rknpu_config = { .bw_priority_addr = 0xfe180008, .bw_priority_length = 0x10, @@ -112,11 +168,16 @@ static const struct rknpu_config rk356x_rknpu_config = { .pc_task_number_mask = 0xfff, .pc_task_status_offset = 0x3c, .pc_dma_ctrl = 0, - .bw_enable = 1, .irqs = rknpu_irqs, - .resets = rknpu_resets, .num_irqs = ARRAY_SIZE(rknpu_irqs), - .num_resets = ARRAY_SIZE(rknpu_resets) + .nbuf_phyaddr = 0, + .nbuf_size = 0, + .max_submit_number = (1 << 12) - 1, + .core_mask = 0x1, + .amount_top = &rknpu_old_top_amount, + .amount_core = NULL, + .state_init = NULL, + .cache_sgt_init = NULL, }; static const struct rknpu_config rk3588_rknpu_config = { @@ -128,11 +189,37 @@ static const struct rknpu_config rk3588_rknpu_config = { .pc_task_number_mask = 0xfff, .pc_task_status_offset = 0x3c, .pc_dma_ctrl = 0, - .bw_enable = 0, .irqs = rk3588_npu_irqs, - .resets = rk3588_npu_resets, .num_irqs = ARRAY_SIZE(rk3588_npu_irqs), - .num_resets = ARRAY_SIZE(rk3588_npu_resets) + .nbuf_phyaddr = 0, + .nbuf_size = 0, + .max_submit_number = (1 << 12) - 1, + .core_mask = 0x7, + .amount_top = NULL, + .amount_core = NULL, + .state_init = NULL, + .cache_sgt_init = NULL, +}; + +static const struct rknpu_config rk3583_rknpu_config = { + .bw_priority_addr = 0x0, + .bw_priority_length = 0x0, + .dma_mask = DMA_BIT_MASK(40), + .pc_data_amount_scale = 2, + .pc_task_number_bits = 12, + .pc_task_number_mask = 0xfff, + .pc_task_status_offset = 0x3c, + .pc_dma_ctrl = 0, + .irqs = rk3588_npu_irqs, + .num_irqs = 2, + .nbuf_phyaddr = 0, + .nbuf_size = 0, + .max_submit_number = (1 << 12) - 1, + .core_mask = 0x3, + .amount_top = NULL, + .amount_core = NULL, + .state_init = NULL, + .cache_sgt_init = NULL, }; static const struct rknpu_config rv1106_rknpu_config = { @@ -144,11 +231,16 @@ static const struct rknpu_config rv1106_rknpu_config = { .pc_task_number_mask = 0xffff, .pc_task_status_offset = 0x3c, .pc_dma_ctrl = 0, - .bw_enable = 1, .irqs = rknpu_irqs, - .resets = rknpu_resets, .num_irqs = ARRAY_SIZE(rknpu_irqs), - .num_resets = ARRAY_SIZE(rknpu_resets) + .nbuf_phyaddr = 0, + .nbuf_size = 0, + .max_submit_number = (1 << 16) - 1, + .core_mask = 0x1, + .amount_top = &rknpu_old_top_amount, + .amount_core = NULL, + .state_init = NULL, + .cache_sgt_init = NULL, }; static const struct rknpu_config rk3562_rknpu_config = { @@ -160,11 +252,37 @@ static const struct rknpu_config rk3562_rknpu_config = { .pc_task_number_mask = 0xffff, .pc_task_status_offset = 0x48, .pc_dma_ctrl = 1, - .bw_enable = 1, .irqs = rknpu_irqs, - .resets = rknpu_resets, .num_irqs = ARRAY_SIZE(rknpu_irqs), - .num_resets = ARRAY_SIZE(rknpu_resets) + .nbuf_phyaddr = 0xfe400000, + .nbuf_size = 256 * 1024, + .max_submit_number = (1 << 16) - 1, + .core_mask = 0x1, + .amount_top = &rknpu_old_top_amount, + .amount_core = NULL, + .state_init = NULL, + .cache_sgt_init = NULL, +}; + +static const struct rknpu_config rk3576_rknpu_config = { + .bw_priority_addr = 0x0, + .bw_priority_length = 0x0, + .dma_mask = DMA_BIT_MASK(40), + .pc_data_amount_scale = 2, + .pc_task_number_bits = 16, + .pc_task_number_mask = 0xffff, + .pc_task_status_offset = 0x48, + .pc_dma_ctrl = 1, + .irqs = rk3576_npu_irqs, + .num_irqs = ARRAY_SIZE(rk3576_npu_irqs), + .nbuf_phyaddr = 0x3fe80000, + .nbuf_size = 1024 * 1024, + .max_submit_number = (1 << 16) - 1, + .core_mask = 0x3, + .amount_top = &rknpu_top_amount, + .amount_core = &rknpu_core_amount, + .state_init = rk3576_state_init, + .cache_sgt_init = rk3576_cache_sgt_init, }; /* driver probe and init */ @@ -189,6 +307,10 @@ static const struct of_device_id rknpu_of_match[] = { .compatible = "rockchip,rk3562-rknpu", .data = &rk3562_rknpu_config, }, + { + .compatible = "rockchip,rk3576-rknpu", + .data = &rk3576_rknpu_config, + }, {}, }; @@ -204,20 +326,26 @@ static int rknpu_power_off(struct rknpu_device *rknpu_dev); static void rknpu_power_off_delay_work(struct work_struct *power_off_work) { + int ret = 0; struct rknpu_device *rknpu_dev = container_of(to_delayed_work(power_off_work), struct rknpu_device, power_off_work); mutex_lock(&rknpu_dev->power_lock); - if (atomic_dec_if_positive(&rknpu_dev->power_refcount) == 0) - rknpu_power_off(rknpu_dev); + if (atomic_dec_if_positive(&rknpu_dev->power_refcount) == 0) { + ret = rknpu_power_off(rknpu_dev); + if (ret) + atomic_inc(&rknpu_dev->power_refcount); + } mutex_unlock(&rknpu_dev->power_lock); + + if (ret) + rknpu_power_put_delay(rknpu_dev); } int rknpu_power_get(struct rknpu_device *rknpu_dev) { int ret = 0; - cancel_delayed_work(&rknpu_dev->power_off_work); mutex_lock(&rknpu_dev->power_lock); if (atomic_inc_return(&rknpu_dev->power_refcount) == 1) ret = rknpu_power_on(rknpu_dev); @@ -231,15 +359,24 @@ int rknpu_power_put(struct rknpu_device *rknpu_dev) int ret = 0; mutex_lock(&rknpu_dev->power_lock); - if (atomic_dec_if_positive(&rknpu_dev->power_refcount) == 0) + if (atomic_dec_if_positive(&rknpu_dev->power_refcount) == 0) { ret = rknpu_power_off(rknpu_dev); + if (ret) + atomic_inc(&rknpu_dev->power_refcount); + } mutex_unlock(&rknpu_dev->power_lock); + if (ret) + rknpu_power_put_delay(rknpu_dev); + return ret; } -static int rknpu_power_put_delay(struct rknpu_device *rknpu_dev) +int rknpu_power_put_delay(struct rknpu_device *rknpu_dev) { + if (rknpu_dev->power_put_delay == 0) + return rknpu_power_put(rknpu_dev); + mutex_lock(&rknpu_dev->power_lock); if (atomic_read(&rknpu_dev->power_refcount) == 1) queue_delayed_work( @@ -248,6 +385,7 @@ static int rknpu_power_put_delay(struct rknpu_device *rknpu_dev) else atomic_dec_if_positive(&rknpu_dev->power_refcount); mutex_unlock(&rknpu_dev->power_lock); + return 0; } @@ -342,6 +480,18 @@ static int rknpu_action(struct rknpu_device *rknpu_dev, args->value = 0; ret = 0; break; + case RKNPU_GET_IOMMU_DOMAIN_ID: + args->value = rknpu_dev->iommu_domain_id; + ret = 0; + break; + case RKNPU_SET_IOMMU_DOMAIN_ID: { + ret = rknpu_iommu_domain_get_and_switch( + rknpu_dev, *(int32_t *)&args->value); + if (ret) + break; + rknpu_iommu_domain_put(rknpu_dev); + break; + } default: ret = -EINVAL; break; @@ -353,11 +503,56 @@ static int rknpu_action(struct rknpu_device *rknpu_dev, #ifdef CONFIG_ROCKCHIP_RKNPU_DMA_HEAP static int rknpu_open(struct inode *inode, struct file *file) { + struct rknpu_device *rknpu_dev = + container_of(file->private_data, struct rknpu_device, miscdev); + struct rknpu_session *session = NULL; + + session = kzalloc(sizeof(*session), GFP_KERNEL); + if (!session) { + LOG_ERROR("rknpu session alloc failed\n"); + return -ENOMEM; + } + + session->rknpu_dev = rknpu_dev; + INIT_LIST_HEAD(&session->list); + + file->private_data = (void *)session; + return nonseekable_open(inode, file); } static int rknpu_release(struct inode *inode, struct file *file) { + struct rknpu_mem_object *entry; + struct rknpu_session *session = file->private_data; + struct rknpu_device *rknpu_dev = session->rknpu_dev; + LIST_HEAD(local_list); + + spin_lock(&rknpu_dev->lock); + list_replace_init(&session->list, &local_list); + file->private_data = NULL; + spin_unlock(&rknpu_dev->lock); + + while (!list_empty(&local_list)) { + entry = list_first_entry(&local_list, struct rknpu_mem_object, + head); + + LOG_DEBUG( + "Fd close free rknpu_obj: %#llx, rknpu_obj->dma_addr: %#llx\n", + (__u64)(uintptr_t)entry, (__u64)entry->dma_addr); + + vunmap(entry->kv_addr); + entry->kv_addr = NULL; + + if (!entry->owner) + dma_buf_put(entry->dmabuf); + + list_del(&entry->head); + kfree(entry); + } + + kfree(session); + return 0; } @@ -386,30 +581,34 @@ static int rknpu_action_ioctl(struct rknpu_device *rknpu_dev, return ret; } -static long rknpu_ioctl(struct file *file, uint32_t cmd, unsigned long arg) +static long rknpu_ioctl(struct file *file, unsigned int cmd, unsigned long arg) { long ret = -EINVAL; - struct rknpu_device *rknpu_dev = - container_of(file->private_data, struct rknpu_device, miscdev); + struct rknpu_device *rknpu_dev = NULL; + + if (!file->private_data) + return -EINVAL; + + rknpu_dev = ((struct rknpu_session *)file->private_data)->rknpu_dev; rknpu_power_get(rknpu_dev); - switch (cmd) { - case IOCTL_RKNPU_ACTION: + switch (_IOC_NR(cmd)) { + case RKNPU_ACTION: ret = rknpu_action_ioctl(rknpu_dev, arg); break; - case IOCTL_RKNPU_SUBMIT: + case RKNPU_SUBMIT: ret = rknpu_submit_ioctl(rknpu_dev, arg); break; - case IOCTL_RKNPU_MEM_CREATE: - ret = rknpu_mem_create_ioctl(rknpu_dev, arg); + case RKNPU_MEM_CREATE: + ret = rknpu_mem_create_ioctl(rknpu_dev, file, cmd, arg); break; case RKNPU_MEM_MAP: break; - case IOCTL_RKNPU_MEM_DESTROY: - ret = rknpu_mem_destroy_ioctl(rknpu_dev, arg); + case RKNPU_MEM_DESTROY: + ret = rknpu_mem_destroy_ioctl(rknpu_dev, file, arg); break; - case IOCTL_RKNPU_MEM_SYNC: + case RKNPU_MEM_SYNC: ret = rknpu_mem_sync_ioctl(rknpu_dev, arg); break; default: @@ -432,11 +631,13 @@ const struct file_operations rknpu_fops = { #endif #ifdef CONFIG_ROCKCHIP_RKNPU_DRM_GEM +#if KERNEL_VERSION(6, 1, 0) > LINUX_VERSION_CODE static const struct vm_operations_struct rknpu_gem_vm_ops = { .fault = rknpu_gem_fault, .open = drm_gem_vm_open, .close = drm_gem_vm_close, }; +#endif static int rknpu_action_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv) @@ -446,16 +647,16 @@ static int rknpu_action_ioctl(struct drm_device *dev, void *data, return rknpu_action(rknpu_dev, (struct rknpu_action *)data); } -#define RKNPU_IOCTL(func) \ - static int __##func(struct drm_device *dev, void *data, \ - struct drm_file *file_priv) \ - { \ - struct rknpu_device *rknpu_dev = dev_get_drvdata(dev->dev); \ - int ret = -EINVAL; \ - rknpu_power_get(rknpu_dev); \ - ret = func(dev, data, file_priv); \ - rknpu_power_put_delay(rknpu_dev); \ - return ret; \ +#define RKNPU_IOCTL(func) \ + static int __##func(struct drm_device *dev, void *data, \ + struct drm_file *file_priv) \ + { \ + struct rknpu_device *rknpu_dev = dev_get_drvdata(dev->dev); \ + int ret = -EINVAL; \ + rknpu_power_get(rknpu_dev); \ + ret = func(dev, data, file_priv); \ + rknpu_power_put_delay(rknpu_dev); \ + return ret; \ } RKNPU_IOCTL(rknpu_action_ioctl); @@ -478,6 +679,9 @@ static const struct drm_ioctl_desc rknpu_ioctls[] = { DRM_RENDER_ALLOW), }; +#if KERNEL_VERSION(6, 1, 0) <= LINUX_VERSION_CODE +DEFINE_DRM_GEM_FOPS(rknpu_drm_driver_fops); +#else static const struct file_operations rknpu_drm_driver_fops = { .owner = THIS_MODULE, .open = drm_open, @@ -491,6 +695,7 @@ static const struct file_operations rknpu_drm_driver_fops = { .release = drm_release, .llseek = noop_llseek, }; +#endif static struct drm_driver rknpu_drm_driver = { #if KERNEL_VERSION(5, 4, 0) <= LINUX_VERSION_CODE @@ -498,28 +703,34 @@ static struct drm_driver rknpu_drm_driver = { #else .driver_features = DRIVER_GEM | DRIVER_PRIME | DRIVER_RENDER, #endif +#if KERNEL_VERSION(6, 1, 0) > LINUX_VERSION_CODE .gem_free_object_unlocked = rknpu_gem_free_object, .gem_vm_ops = &rknpu_gem_vm_ops, + .dumb_destroy = drm_gem_dumb_destroy, + .gem_prime_export = drm_gem_prime_export, + .gem_prime_get_sg_table = rknpu_gem_prime_get_sg_table, + .gem_prime_vmap = rknpu_gem_prime_vmap, + .gem_prime_vunmap = rknpu_gem_prime_vunmap, +#endif .dumb_create = rknpu_gem_dumb_create, #if KERNEL_VERSION(4, 19, 0) > LINUX_VERSION_CODE .dumb_map_offset = rknpu_gem_dumb_map_offset, #else .dumb_map_offset = drm_gem_dumb_map_offset, #endif - .dumb_destroy = drm_gem_dumb_destroy, .prime_handle_to_fd = drm_gem_prime_handle_to_fd, .prime_fd_to_handle = drm_gem_prime_fd_to_handle, - .gem_prime_export = drm_gem_prime_export, #if KERNEL_VERSION(4, 13, 0) <= LINUX_VERSION_CODE .gem_prime_import = rknpu_gem_prime_import, #else .gem_prime_import = drm_gem_prime_import, #endif - .gem_prime_get_sg_table = rknpu_gem_prime_get_sg_table, .gem_prime_import_sg_table = rknpu_gem_prime_import_sg_table, - .gem_prime_vmap = rknpu_gem_prime_vmap, - .gem_prime_vunmap = rknpu_gem_prime_vunmap, +#if KERNEL_VERSION(6, 1, 0) <= LINUX_VERSION_CODE + .gem_prime_mmap = drm_gem_prime_mmap, +#else .gem_prime_mmap = rknpu_gem_prime_mmap, +#endif .ioctls = rknpu_ioctls, .num_ioctls = ARRAY_SIZE(rknpu_ioctls), .fops = &rknpu_drm_driver_fops, @@ -539,7 +750,7 @@ static enum hrtimer_restart hrtimer_handler(struct hrtimer *timer) container_of(timer, struct rknpu_device, timer); struct rknpu_subcore_data *subcore_data = NULL; struct rknpu_job *job = NULL; - ktime_t now = ktime_get(); + ktime_t now; unsigned long flags; int i; @@ -550,14 +761,16 @@ static enum hrtimer_restart hrtimer_handler(struct hrtimer *timer) job = subcore_data->job; if (job) { + now = ktime_get(); subcore_data->timer.busy_time += - ktime_us_delta(now, job->hw_recoder_time); - job->hw_recoder_time = ktime_get(); + ktime_sub(now, job->hw_recoder_time); + job->hw_recoder_time = now; } - subcore_data->timer.busy_time_record = + subcore_data->timer.total_busy_time = subcore_data->timer.busy_time; subcore_data->timer.busy_time = 0; + spin_unlock_irqrestore(&rknpu_dev->irq_lock, flags); } @@ -604,6 +817,42 @@ static bool rknpu_is_iommu_enable(struct device *dev) } #ifdef CONFIG_ROCKCHIP_RKNPU_DRM_GEM +static int drm_fake_dev_register(struct rknpu_device *rknpu_dev) +{ + const struct platform_device_info rknpu_dev_info = { + .name = "rknpu_dev", + .id = PLATFORM_DEVID_AUTO, + .dma_mask = rknpu_dev->config->dma_mask, + }; + struct platform_device *pdev = NULL; + int ret = -EINVAL; + + pdev = platform_device_register_full(&rknpu_dev_info); + if (pdev) { + ret = of_dma_configure(&pdev->dev, NULL, true); + if (ret) { + platform_device_unregister(pdev); + pdev = NULL; + } + } + + rknpu_dev->fake_dev = pdev ? &pdev->dev : NULL; + + return ret; +} + +static void drm_fake_dev_unregister(struct rknpu_device *rknpu_dev) +{ + struct platform_device *pdev = NULL; + + if (!rknpu_dev->fake_dev) + return; + + pdev = to_platform_device(rknpu_dev->fake_dev); + + platform_device_unregister(pdev); +} + static int rknpu_drm_probe(struct rknpu_device *rknpu_dev) { struct device *dev = rknpu_dev->dev; @@ -622,6 +871,8 @@ static int rknpu_drm_probe(struct rknpu_device *rknpu_dev) drm_dev->dev_private = rknpu_dev; rknpu_dev->drm_dev = drm_dev; + drm_fake_dev_register(rknpu_dev); + return 0; err_free_drm: @@ -638,6 +889,8 @@ static void rknpu_drm_remove(struct rknpu_device *rknpu_dev) { struct drm_device *drm_dev = rknpu_dev->drm_dev; + drm_fake_dev_unregister(rknpu_dev); + drm_dev_unregister(drm_dev); #if KERNEL_VERSION(4, 15, 0) <= LINUX_VERSION_CODE @@ -685,9 +938,7 @@ static int rknpu_power_on(struct rknpu_device *rknpu_dev) } #ifndef FPGA_PLATFORM -#if KERNEL_VERSION(5, 10, 0) <= LINUX_VERSION_CODE - rockchip_monitor_volt_adjust_lock(rknpu_dev->mdev_info); -#endif + rknpu_devfreq_lock(rknpu_dev); #endif if (rknpu_dev->multiple_domains) { @@ -744,11 +995,12 @@ static int rknpu_power_on(struct rknpu_device *rknpu_dev) ret); } + if (rknpu_dev->config->state_init != NULL) + rknpu_dev->config->state_init(rknpu_dev); + out: #ifndef FPGA_PLATFORM -#if KERNEL_VERSION(5, 10, 0) <= LINUX_VERSION_CODE - rockchip_monitor_volt_adjust_unlock(rknpu_dev->mdev_info); -#endif + rknpu_devfreq_unlock(rknpu_dev); #endif return ret; @@ -762,9 +1014,7 @@ static int rknpu_power_off(struct rknpu_device *rknpu_dev) int ret; bool val; -#if KERNEL_VERSION(5, 10, 0) <= LINUX_VERSION_CODE - rockchip_monitor_volt_adjust_lock(rknpu_dev->mdev_info); -#endif + rknpu_devfreq_lock(rknpu_dev); #endif pm_runtime_put_sync(dev); @@ -786,10 +1036,7 @@ static int rknpu_power_off(struct rknpu_device *rknpu_dev) if (ret) { LOG_DEV_ERROR(dev, "iommu still enabled\n"); pm_runtime_get_sync(dev); -#if KERNEL_VERSION(5, 10, 0) <= LINUX_VERSION_CODE - rockchip_monitor_volt_adjust_unlock( - rknpu_dev->mdev_info); -#endif + rknpu_devfreq_unlock(rknpu_dev); return ret; } #else @@ -805,9 +1052,7 @@ static int rknpu_power_off(struct rknpu_device *rknpu_dev) } #ifndef FPGA_PLATFORM -#if KERNEL_VERSION(5, 10, 0) <= LINUX_VERSION_CODE - rockchip_monitor_volt_adjust_unlock(rknpu_dev->mdev_info); -#endif + rknpu_devfreq_unlock(rknpu_dev); #endif clk_bulk_disable_unprepare(rknpu_dev->num_clks, rknpu_dev->clks); @@ -823,694 +1068,17 @@ static int rknpu_power_off(struct rknpu_device *rknpu_dev) return 0; } -#ifndef FPGA_PLATFORM -static struct monitor_dev_profile npu_mdevp = { - .type = MONITOR_TPYE_DEV, - .low_temp_adjust = rockchip_monitor_dev_low_temp_adjust, - .high_temp_adjust = rockchip_monitor_dev_high_temp_adjust, -#if KERNEL_VERSION(5, 10, 0) <= LINUX_VERSION_CODE - .update_volt = rockchip_monitor_check_rate_volt, -#endif -}; - -#if KERNEL_VERSION(5, 10, 0) <= LINUX_VERSION_CODE -static int npu_opp_helper(struct dev_pm_set_opp_data *data) -{ - struct device *dev = data->dev; - struct dev_pm_opp_supply *old_supply_vdd = &data->old_opp.supplies[0]; - struct dev_pm_opp_supply *old_supply_mem = &data->old_opp.supplies[1]; - struct dev_pm_opp_supply *new_supply_vdd = &data->new_opp.supplies[0]; - struct dev_pm_opp_supply *new_supply_mem = &data->new_opp.supplies[1]; - struct regulator *vdd_reg = data->regulators[0]; - struct regulator *mem_reg = data->regulators[1]; - struct clk *clk = data->clk; - struct rknpu_device *rknpu_dev = dev_get_drvdata(dev); - struct rockchip_opp_info *opp_info = &rknpu_dev->opp_info; - unsigned long old_freq = data->old_opp.rate; - unsigned long new_freq = data->new_opp.rate; - bool is_set_rm = true; - bool is_set_clk = true; - u32 target_rm = UINT_MAX; - int ret = 0; - - if (!pm_runtime_active(dev)) { - is_set_rm = false; - if (opp_info->scmi_clk) - is_set_clk = false; - } - - ret = clk_bulk_prepare_enable(opp_info->num_clks, opp_info->clks); - if (ret < 0) { - LOG_DEV_ERROR(dev, "failed to enable opp clks\n"); - return ret; - } - rockchip_get_read_margin(dev, opp_info, new_supply_vdd->u_volt, - &target_rm); - - /* Change frequency */ - LOG_DEV_DEBUG(dev, "switching OPP: %lu Hz --> %lu Hz\n", old_freq, - new_freq); - /* Scaling up? Scale voltage before frequency */ - if (new_freq >= old_freq) { - rockchip_set_intermediate_rate(dev, opp_info, clk, old_freq, - new_freq, true, is_set_clk); - ret = regulator_set_voltage(mem_reg, new_supply_mem->u_volt, - INT_MAX); - if (ret) { - LOG_DEV_ERROR(dev, - "failed to set volt %lu uV for mem reg\n", - new_supply_mem->u_volt); - goto restore_voltage; - } - ret = regulator_set_voltage(vdd_reg, new_supply_vdd->u_volt, - INT_MAX); - if (ret) { - LOG_DEV_ERROR(dev, - "failed to set volt %lu uV for vdd reg\n", - new_supply_vdd->u_volt); - goto restore_voltage; - } - rockchip_set_read_margin(dev, opp_info, target_rm, is_set_rm); - if (is_set_clk && clk_set_rate(clk, new_freq)) { - ret = -EINVAL; - LOG_DEV_ERROR(dev, "failed to set clk rate: %d\n", ret); - goto restore_rm; - } - /* Scaling down? Scale voltage after frequency */ - } else { - rockchip_set_intermediate_rate(dev, opp_info, clk, old_freq, - new_freq, false, is_set_clk); - rockchip_set_read_margin(dev, opp_info, target_rm, is_set_rm); - if (is_set_clk && clk_set_rate(clk, new_freq)) { - ret = -EINVAL; - LOG_DEV_ERROR(dev, "failed to set clk rate: %d\n", ret); - goto restore_rm; - } - ret = regulator_set_voltage(vdd_reg, new_supply_vdd->u_volt, - INT_MAX); - if (ret) { - LOG_DEV_ERROR(dev, - "failed to set volt %lu uV for vdd reg\n", - new_supply_vdd->u_volt); - goto restore_freq; - } - ret = regulator_set_voltage(mem_reg, new_supply_mem->u_volt, - INT_MAX); - if (ret) { - LOG_DEV_ERROR(dev, - "failed to set volt %lu uV for mem reg\n", - new_supply_mem->u_volt); - goto restore_freq; - } - } - - clk_bulk_disable_unprepare(opp_info->num_clks, opp_info->clks); - - return 0; - -restore_freq: - if (is_set_clk && clk_set_rate(clk, old_freq)) - LOG_DEV_ERROR(dev, "failed to restore old-freq %lu Hz\n", - old_freq); -restore_rm: - rockchip_get_read_margin(dev, opp_info, old_supply_vdd->u_volt, - &target_rm); - rockchip_set_read_margin(dev, opp_info, opp_info->current_rm, - is_set_rm); -restore_voltage: - regulator_set_voltage(mem_reg, old_supply_mem->u_volt, INT_MAX); - regulator_set_voltage(vdd_reg, old_supply_vdd->u_volt, INT_MAX); - clk_bulk_disable_unprepare(opp_info->num_clks, opp_info->clks); - - return ret; -} - -static int npu_devfreq_target(struct device *dev, unsigned long *freq, - u32 flags) -{ - struct rknpu_device *rknpu_dev = dev_get_drvdata(dev); - struct dev_pm_opp *opp; - unsigned long opp_volt; - int ret = 0; - - if (!npu_mdevp.is_checked) - return -EINVAL; - - opp = devfreq_recommended_opp(dev, freq, flags); - if (IS_ERR(opp)) - return PTR_ERR(opp); - opp_volt = dev_pm_opp_get_voltage(opp); - dev_pm_opp_put(opp); - -#if KERNEL_VERSION(5, 10, 0) <= LINUX_VERSION_CODE - rockchip_monitor_volt_adjust_lock(rknpu_dev->mdev_info); -#endif - ret = dev_pm_opp_set_rate(dev, *freq); - if (!ret) { - rknpu_dev->current_freq = *freq; - if (rknpu_dev->devfreq) - rknpu_dev->devfreq->last_status.current_frequency = - *freq; - rknpu_dev->current_volt = opp_volt; - LOG_DEV_INFO(dev, "set rknpu freq: %lu, volt: %lu\n", - rknpu_dev->current_freq, rknpu_dev->current_volt); - } -#if KERNEL_VERSION(5, 10, 0) <= LINUX_VERSION_CODE - rockchip_monitor_volt_adjust_unlock(rknpu_dev->mdev_info); -#endif - - return ret; -} - -#else - -static int npu_devfreq_target(struct device *dev, unsigned long *target_freq, - u32 flags) -{ - struct rknpu_device *rknpu_dev = dev_get_drvdata(dev); - struct dev_pm_opp *opp = NULL; - unsigned long freq = *target_freq; - unsigned long old_freq = rknpu_dev->current_freq; - unsigned long volt, old_volt = rknpu_dev->current_volt; - int ret = -EINVAL; - -#if KERNEL_VERSION(4, 11, 0) > LINUX_VERSION_CODE - rcu_read_lock(); -#endif - - opp = devfreq_recommended_opp(dev, &freq, flags); - if (IS_ERR(opp)) { -#if KERNEL_VERSION(4, 11, 0) > LINUX_VERSION_CODE - rcu_read_unlock(); -#endif - LOG_DEV_ERROR(dev, "failed to get opp (%ld)\n", PTR_ERR(opp)); - return PTR_ERR(opp); - } - volt = dev_pm_opp_get_voltage(opp); -#if KERNEL_VERSION(4, 11, 0) > LINUX_VERSION_CODE - rcu_read_unlock(); -#endif - - /* - * Only update if there is a change of frequency - */ - if (old_freq == freq) { - *target_freq = freq; - if (old_volt == volt) - return 0; - ret = regulator_set_voltage(rknpu_dev->vdd, volt, INT_MAX); - if (ret) { - LOG_DEV_ERROR(dev, "failed to set volt %lu\n", volt); - return ret; - } - rknpu_dev->current_volt = volt; - return 0; - } - - if (rknpu_dev->vdd && old_volt != volt && old_freq < freq) { - ret = regulator_set_voltage(rknpu_dev->vdd, volt, INT_MAX); - if (ret) { - LOG_DEV_ERROR(dev, "failed to increase volt %lu\n", - volt); - return ret; - } - } - LOG_DEV_DEBUG(dev, "%luHz %luuV -> %luHz %luuV\n", old_freq, old_volt, - freq, volt); - ret = clk_set_rate(rknpu_dev->clks[0].clk, freq); - if (ret) { - LOG_DEV_ERROR(dev, "failed to set clock %lu\n", freq); - return ret; - } - *target_freq = freq; - rknpu_dev->current_freq = freq; - - if (rknpu_dev->devfreq) - rknpu_dev->devfreq->last_status.current_frequency = freq; - - if (rknpu_dev->vdd && old_volt != volt && old_freq > freq) { - ret = regulator_set_voltage(rknpu_dev->vdd, volt, INT_MAX); - if (ret) { - LOG_DEV_ERROR(dev, "failed to decrease volt %lu\n", - volt); - return ret; - } - } - rknpu_dev->current_volt = volt; - - LOG_DEV_INFO(dev, "set rknpu freq: %lu, volt: %lu\n", - rknpu_dev->current_freq, rknpu_dev->current_volt); - - return ret; -} -#endif - -static int npu_devfreq_get_dev_status(struct device *dev, - struct devfreq_dev_status *stat) -{ - return 0; -} - -static int npu_devfreq_get_cur_freq(struct device *dev, unsigned long *freq) -{ - struct rknpu_device *rknpu_dev = dev_get_drvdata(dev); - - *freq = rknpu_dev->current_freq; - - return 0; -} - -static struct devfreq_dev_profile npu_devfreq_profile = { - .polling_ms = 50, - .target = npu_devfreq_target, - .get_dev_status = npu_devfreq_get_dev_status, - .get_cur_freq = npu_devfreq_get_cur_freq, -}; - -#ifdef CONFIG_PM_DEVFREQ -static int devfreq_rknpu_ondemand_func(struct devfreq *df, unsigned long *freq) -{ - struct rknpu_device *rknpu_dev = df->data; - - if (rknpu_dev) - *freq = rknpu_dev->ondemand_freq; - else - *freq = df->previous_freq; - - return 0; -} - -static int devfreq_rknpu_ondemand_handler(struct devfreq *devfreq, - unsigned int event, void *data) -{ - return 0; -} - -static struct devfreq_governor devfreq_rknpu_ondemand = { - .name = "rknpu_ondemand", - .get_target_freq = devfreq_rknpu_ondemand_func, - .event_handler = devfreq_rknpu_ondemand_handler, -}; -#endif - -static unsigned long npu_get_static_power(struct devfreq *devfreq, - unsigned long voltage) -{ - struct device *dev = devfreq->dev.parent; - struct rknpu_device *rknpu_dev = dev_get_drvdata(dev); - - if (!rknpu_dev->model_data) - return 0; - - return rockchip_ipa_get_static_power(rknpu_dev->model_data, voltage); -} - -static struct devfreq_cooling_power npu_cooling_power = { - .get_static_power = &npu_get_static_power, -}; - -#if KERNEL_VERSION(5, 10, 0) <= LINUX_VERSION_CODE -static int rk3588_npu_get_soc_info(struct device *dev, struct device_node *np, - int *bin, int *process) -{ - int ret = 0; - u8 value = 0; - - if (!bin) - return 0; - - if (of_property_match_string(np, "nvmem-cell-names", - "specification_serial_number") >= 0) { - ret = rockchip_nvmem_cell_read_u8( - np, "specification_serial_number", &value); - if (ret) { - LOG_DEV_ERROR( - dev, - "Failed to get specification_serial_number\n"); - return ret; - } - /* RK3588M */ - if (value == 0xd) - *bin = 1; - /* RK3588J */ - else if (value == 0xa) - *bin = 2; - } - if (*bin < 0) - *bin = 0; - LOG_DEV_INFO(dev, "bin=%d\n", *bin); - - return ret; -} - -static int rk3588_npu_set_soc_info(struct device *dev, struct device_node *np, - int bin, int process, int volt_sel) -{ - struct opp_table *opp_table; - u32 supported_hw[2]; - - if (volt_sel < 0) - return 0; - if (bin < 0) - bin = 0; - - if (!of_property_read_bool(np, "rockchip,supported-hw")) - return 0; - - /* SoC Version */ - supported_hw[0] = BIT(bin); - /* Speed Grade */ - supported_hw[1] = BIT(volt_sel); - opp_table = dev_pm_opp_set_supported_hw(dev, supported_hw, 2); - if (IS_ERR(opp_table)) { - LOG_DEV_ERROR(dev, "failed to set supported opp\n"); - return PTR_ERR(opp_table); - } - - return 0; -} - -static int rk3588_npu_set_read_margin(struct device *dev, - struct rockchip_opp_info *opp_info, - u32 rm) -{ - u32 offset = 0, val = 0; - int i, ret = 0; - - if (!opp_info->grf || !opp_info->volt_rm_tbl) - return 0; - - if (rm == opp_info->current_rm || rm == UINT_MAX) - return 0; - - LOG_DEV_DEBUG(dev, "set rm to %d\n", rm); - - for (i = 0; i < 3; i++) { - ret = regmap_read(opp_info->grf, offset, &val); - if (ret < 0) { - LOG_DEV_ERROR(dev, "failed to get rm from 0x%x\n", - offset); - return ret; - } - val &= ~0x1c; - regmap_write(opp_info->grf, offset, val | (rm << 2)); - offset += 4; - } - opp_info->current_rm = rm; - - return 0; -} - -static const struct rockchip_opp_data rk3588_npu_opp_data = { - .get_soc_info = rk3588_npu_get_soc_info, - .set_soc_info = rk3588_npu_set_soc_info, - .set_read_margin = rk3588_npu_set_read_margin, -}; - -static const struct of_device_id rockchip_npu_of_match[] = { - { - .compatible = "rockchip,rk3588", - .data = (void *)&rk3588_npu_opp_data, - }, - {}, -}; - -static int rknpu_devfreq_init(struct rknpu_device *rknpu_dev) -{ - struct device *dev = rknpu_dev->dev; - struct devfreq_dev_profile *dp = &npu_devfreq_profile; - struct dev_pm_opp *opp; - struct opp_table *reg_table = NULL; - struct opp_table *opp_table = NULL; - const char *const reg_names[] = { "rknpu", "mem" }; - int ret = -EINVAL; - - if (of_find_property(dev->of_node, "rknpu-supply", NULL) && - of_find_property(dev->of_node, "mem-supply", NULL)) { - reg_table = dev_pm_opp_set_regulators(dev, reg_names, 2); - if (IS_ERR(reg_table)) - return PTR_ERR(reg_table); - opp_table = - dev_pm_opp_register_set_opp_helper(dev, npu_opp_helper); - if (IS_ERR(opp_table)) { - dev_pm_opp_put_regulators(reg_table); - return PTR_ERR(opp_table); - } - } else { - reg_table = dev_pm_opp_set_regulators(dev, reg_names, 1); - if (IS_ERR(reg_table)) - return PTR_ERR(reg_table); - } - - rockchip_get_opp_data(rockchip_npu_of_match, &rknpu_dev->opp_info); - ret = rockchip_init_opp_table(dev, &rknpu_dev->opp_info, "npu_leakage", - "rknpu"); - if (ret) { - LOG_DEV_ERROR(dev, "failed to init_opp_table\n"); - return ret; - } - - rknpu_dev->current_freq = clk_get_rate(rknpu_dev->clks[0].clk); - - opp = devfreq_recommended_opp(dev, &rknpu_dev->current_freq, 0); - if (IS_ERR(opp)) { - ret = PTR_ERR(opp); - goto err_remove_table; - } - dev_pm_opp_put(opp); - dp->initial_freq = rknpu_dev->current_freq; - -#ifdef CONFIG_PM_DEVFREQ - ret = devfreq_add_governor(&devfreq_rknpu_ondemand); - if (ret) { - LOG_DEV_ERROR(dev, "failed to add rknpu_ondemand governor\n"); - goto err_remove_table; - } -#endif - - rknpu_dev->devfreq = devm_devfreq_add_device(dev, dp, "rknpu_ondemand", - (void *)rknpu_dev); - if (IS_ERR(rknpu_dev->devfreq)) { - LOG_DEV_ERROR(dev, "failed to add devfreq\n"); - ret = PTR_ERR(rknpu_dev->devfreq); - goto err_remove_governor; - } - devm_devfreq_register_opp_notifier(dev, rknpu_dev->devfreq); - - rknpu_dev->devfreq->last_status.current_frequency = dp->initial_freq; - rknpu_dev->devfreq->last_status.total_time = 1; - rknpu_dev->devfreq->last_status.busy_time = 1; - - npu_mdevp.data = rknpu_dev->devfreq; - npu_mdevp.opp_info = &rknpu_dev->opp_info; - rknpu_dev->mdev_info = - rockchip_system_monitor_register(dev, &npu_mdevp); - if (IS_ERR(rknpu_dev->mdev_info)) { - LOG_DEV_DEBUG(dev, "without system monitor\n"); - rknpu_dev->mdev_info = NULL; - npu_mdevp.is_checked = true; - } - rknpu_dev->current_freq = clk_get_rate(rknpu_dev->clks[0].clk); - rknpu_dev->current_volt = regulator_get_voltage(rknpu_dev->vdd); - - of_property_read_u32(dev->of_node, "dynamic-power-coefficient", - (u32 *)&npu_cooling_power.dyn_power_coeff); - rknpu_dev->model_data = - rockchip_ipa_power_model_init(dev, "npu_leakage"); - if (IS_ERR_OR_NULL(rknpu_dev->model_data)) { - rknpu_dev->model_data = NULL; - LOG_DEV_ERROR(dev, "failed to initialize power model\n"); - } else if (rknpu_dev->model_data->dynamic_coefficient) { - npu_cooling_power.dyn_power_coeff = - rknpu_dev->model_data->dynamic_coefficient; - } - if (!npu_cooling_power.dyn_power_coeff) { - LOG_DEV_ERROR(dev, "failed to get dynamic-coefficient\n"); - goto out; - } - - rknpu_dev->devfreq_cooling = of_devfreq_cooling_register_power( - dev->of_node, rknpu_dev->devfreq, &npu_cooling_power); - if (IS_ERR_OR_NULL(rknpu_dev->devfreq_cooling)) - LOG_DEV_ERROR(dev, "failed to register cooling device\n"); - -out: - return 0; - -err_remove_governor: -#ifdef CONFIG_PM_DEVFREQ - devfreq_remove_governor(&devfreq_rknpu_ondemand); -#endif -err_remove_table: - dev_pm_opp_of_remove_table(dev); - - rknpu_dev->devfreq = NULL; - - return ret; -} - -#else - -static int npu_devfreq_adjust_current_freq_volt(struct device *dev, - struct rknpu_device *rknpu_dev) -{ - unsigned long volt, old_freq, freq; - struct dev_pm_opp *opp = NULL; - int ret = -EINVAL; - - old_freq = clk_get_rate(rknpu_dev->clks[0].clk); - freq = old_freq; - -#if KERNEL_VERSION(4, 11, 0) > LINUX_VERSION_CODE - rcu_read_lock(); -#endif - - opp = devfreq_recommended_opp(dev, &freq, 0); - volt = dev_pm_opp_get_voltage(opp); - -#if KERNEL_VERSION(4, 11, 0) > LINUX_VERSION_CODE - rcu_read_unlock(); -#endif - - if (freq >= old_freq && rknpu_dev->vdd) { - ret = regulator_set_voltage(rknpu_dev->vdd, volt, INT_MAX); - if (ret) { - LOG_DEV_ERROR(dev, "failed to set volt %lu\n", volt); - return ret; - } - } - LOG_DEV_DEBUG(dev, "adjust current freq=%luHz, volt=%luuV\n", freq, - volt); - ret = clk_set_rate(rknpu_dev->clks[0].clk, freq); - if (ret) { - LOG_DEV_ERROR(dev, "failed to set clock %lu\n", freq); - return ret; - } - if (freq < old_freq && rknpu_dev->vdd) { - ret = regulator_set_voltage(rknpu_dev->vdd, volt, INT_MAX); - if (ret) { - LOG_DEV_ERROR(dev, "failed to set volt %lu\n", volt); - return ret; - } - } - rknpu_dev->current_freq = freq; - rknpu_dev->current_volt = volt; - - return 0; -} - -static int rknpu_devfreq_init(struct rknpu_device *rknpu_dev) -{ - struct device *dev = rknpu_dev->dev; - struct devfreq_dev_profile *dp = &npu_devfreq_profile; - int ret = -EINVAL; - - ret = rockchip_init_opp_table(dev, NULL, "npu_leakage", "rknpu"); - if (ret) { - LOG_DEV_ERROR(dev, "failed to init_opp_table\n"); - return ret; - } - - ret = npu_devfreq_adjust_current_freq_volt(dev, rknpu_dev); - if (ret) { - LOG_DEV_ERROR(dev, "failed to adjust current freq volt\n"); - goto err_remove_table; - } - dp->initial_freq = rknpu_dev->current_freq; - -#ifdef CONFIG_PM_DEVFREQ - ret = devfreq_add_governor(&devfreq_rknpu_ondemand); - if (ret) { - LOG_DEV_ERROR(dev, "failed to add rknpu_ondemand governor\n"); - goto err_remove_table; - } -#endif - - rknpu_dev->devfreq = devm_devfreq_add_device(dev, dp, "rknpu_ondemand", - (void *)rknpu_dev); - if (IS_ERR(rknpu_dev->devfreq)) { - LOG_DEV_ERROR(dev, "failed to add devfreq\n"); - ret = PTR_ERR(rknpu_dev->devfreq); - goto err_remove_governor; - } - devm_devfreq_register_opp_notifier(dev, rknpu_dev->devfreq); - - rknpu_dev->devfreq->last_status.current_frequency = dp->initial_freq; - rknpu_dev->devfreq->last_status.total_time = 1; - rknpu_dev->devfreq->last_status.busy_time = 1; - - npu_mdevp.data = rknpu_dev->devfreq; - rknpu_dev->mdev_info = - rockchip_system_monitor_register(dev, &npu_mdevp); - if (IS_ERR(rknpu_dev->mdev_info)) { - LOG_DEV_DEBUG(dev, "without system monitor\n"); - rknpu_dev->mdev_info = NULL; - } - rknpu_dev->current_freq = clk_get_rate(rknpu_dev->clks[0].clk); - rknpu_dev->current_volt = regulator_get_voltage(rknpu_dev->vdd); - - of_property_read_u32(dev->of_node, "dynamic-power-coefficient", - (u32 *)&npu_cooling_power.dyn_power_coeff); - rknpu_dev->model_data = - rockchip_ipa_power_model_init(dev, "npu_leakage"); - if (IS_ERR_OR_NULL(rknpu_dev->model_data)) { - rknpu_dev->model_data = NULL; - LOG_DEV_ERROR(dev, "failed to initialize power model\n"); - } else if (rknpu_dev->model_data->dynamic_coefficient) { - npu_cooling_power.dyn_power_coeff = - rknpu_dev->model_data->dynamic_coefficient; - } - - if (!npu_cooling_power.dyn_power_coeff) { - LOG_DEV_ERROR(dev, "failed to get dynamic-coefficient\n"); - goto out; - } - - rknpu_dev->devfreq_cooling = of_devfreq_cooling_register_power( - dev->of_node, rknpu_dev->devfreq, &npu_cooling_power); - if (IS_ERR_OR_NULL(rknpu_dev->devfreq_cooling)) - LOG_DEV_ERROR(dev, "failed to register cooling device\n"); - -out: - return 0; - -err_remove_governor: -#ifdef CONFIG_PM_DEVFREQ - devfreq_remove_governor(&devfreq_rknpu_ondemand); -#endif -err_remove_table: - dev_pm_opp_of_remove_table(dev); - - rknpu_dev->devfreq = NULL; - - return ret; -} -#endif - -static int rknpu_devfreq_remove(struct rknpu_device *rknpu_dev) -{ - if (rknpu_dev->devfreq) { - devfreq_unregister_opp_notifier(rknpu_dev->dev, - rknpu_dev->devfreq); - dev_pm_opp_of_remove_table(rknpu_dev->dev); -#ifdef CONFIG_PM_DEVFREQ - devfreq_remove_governor(&devfreq_rknpu_ondemand); -#endif - } - - return 0; -} - -#endif - static int rknpu_register_irq(struct platform_device *pdev, struct rknpu_device *rknpu_dev) { const struct rknpu_config *config = rknpu_dev->config; struct device *dev = &pdev->dev; +#if KERNEL_VERSION(6, 1, 0) > LINUX_VERSION_CODE struct resource *res; +#endif int i, ret, irq; +#if KERNEL_VERSION(6, 1, 0) > LINUX_VERSION_CODE res = platform_get_resource_byname(pdev, IORESOURCE_IRQ, config->irqs[0].name); if (res) { @@ -1549,6 +1117,28 @@ static int rknpu_register_irq(struct platform_device *pdev, return ret; } } +#else + /* there are irq names in dts */ + for (i = 0; i < config->num_irqs; i++) { + irq = platform_get_irq_byname(pdev, config->irqs[i].name); + if (irq < 0) { + irq = platform_get_irq(pdev, i); + if (irq < 0) { + LOG_DEV_ERROR(dev, "no npu %s in dts\n", + config->irqs[i].name); + return irq; + } + } + + ret = devm_request_irq(dev, irq, config->irqs[i].irq_hdl, + IRQF_SHARED, dev_name(dev), rknpu_dev); + if (ret < 0) { + LOG_DEV_ERROR(dev, "request %s failed: %d\n", + config->irqs[i].name, ret); + return ret; + } + } +#endif return 0; } @@ -1603,6 +1193,53 @@ static int rknpu_find_sram_resource(struct rknpu_device *rknpu_dev) return 0; } +static int rknpu_find_nbuf_resource(struct rknpu_device *rknpu_dev) +{ + struct device *dev = rknpu_dev->dev; + + if (rknpu_dev->config->nbuf_size == 0) + return -EINVAL; + + rknpu_dev->nbuf_start = rknpu_dev->config->nbuf_phyaddr; + rknpu_dev->nbuf_size = rknpu_dev->config->nbuf_size; + rknpu_dev->nbuf_base_io = + devm_ioremap(dev, rknpu_dev->nbuf_start, rknpu_dev->nbuf_size); + if (IS_ERR(rknpu_dev->nbuf_base_io)) { + LOG_DEV_ERROR(dev, "failed to remap nbuf base io!\n"); + rknpu_dev->nbuf_base_io = NULL; + } + + rknpu_dev->nbuf_end = rknpu_dev->nbuf_start + rknpu_dev->nbuf_size; + + LOG_DEV_INFO(dev, "nbuf region: [%pa, %pa), nbuf size: %#x\n", + &rknpu_dev->nbuf_start, &rknpu_dev->nbuf_end, + rknpu_dev->nbuf_size); + + return 0; +} + +static int rknpu_get_invalid_core_mask(struct device *dev) +{ + int ret = 0; + u8 invalid_core_mask = 0; + + if (of_property_match_string(dev->of_node, "nvmem-cell-names", + "cores") >= 0) { + ret = rockchip_nvmem_cell_read_u8(dev->of_node, "cores", + &invalid_core_mask); + /* The default valid npu cores for RK3583 are core0 and core1 */ + invalid_core_mask |= RKNPU_CORE2_MASK; + if (ret) { + LOG_DEV_ERROR( + dev, + "failed to get specification_serial_number\n"); + return invalid_core_mask; + } + } + + return (int)invalid_core_mask; +} + static int rknpu_probe(struct platform_device *pdev) { struct resource *res = NULL; @@ -1634,11 +1271,32 @@ static int rknpu_probe(struct platform_device *pdev) if (!config) return -EINVAL; + if (match->data == (void *)&rk3588_rknpu_config) { + int invalid_core_mask = rknpu_get_invalid_core_mask(dev); + /* The default valid npu cores for RK3583 are core0 and core1 */ + if (invalid_core_mask & RKNPU_CORE2_MASK) { + if ((invalid_core_mask & RKNPU_CORE0_MASK) || + (invalid_core_mask & RKNPU_CORE1_MASK)) { + LOG_DEV_ERROR( + dev, + "rknpu core invalid, invalid core mask: %#x\n", + invalid_core_mask); + return -ENODEV; + } + config = &rk3583_rknpu_config; + } + } + rknpu_dev->config = config; rknpu_dev->dev = dev; + dev_set_drvdata(dev, rknpu_dev); rknpu_dev->iommu_en = rknpu_is_iommu_enable(dev); - if (!rknpu_dev->iommu_en) { + if (rknpu_dev->iommu_en) { + rknpu_dev->iommu_group = iommu_group_get(dev); + if (!rknpu_dev->iommu_group) + return -EINVAL; + } else { /* Initialize reserved memory resources */ ret = of_reserved_mem_device_init(dev); if (!ret) { @@ -1662,11 +1320,6 @@ static int rknpu_probe(struct platform_device *pdev) } #ifndef FPGA_PLATFORM -#if KERNEL_VERSION(5, 10, 0) <= LINUX_VERSION_CODE - if (strstr(__clk_get_name(rknpu_dev->clks[0].clk), "scmi")) - rknpu_dev->opp_info.scmi_clk = rknpu_dev->clks[0].clk; -#endif - rknpu_dev->vdd = devm_regulator_get_optional(dev, "rknpu"); if (IS_ERR(rknpu_dev->vdd)) { if (PTR_ERR(rknpu_dev->vdd) != -ENODEV) { @@ -1698,6 +1351,7 @@ static int rknpu_probe(struct platform_device *pdev) spin_lock_init(&rknpu_dev->irq_lock); mutex_init(&rknpu_dev->power_lock); mutex_init(&rknpu_dev->reset_lock); + mutex_init(&rknpu_dev->domain_lock); for (i = 0; i < config->num_irqs; i++) { INIT_LIST_HEAD(&rknpu_dev->subcore_datas[i].todo_list); init_waitqueue_head(&rknpu_dev->subcore_datas[i].job_done_wq); @@ -1793,9 +1447,11 @@ static int rknpu_probe(struct platform_device *pdev) virt_dev = dev_pm_domain_attach_by_name(dev, "npu1"); if (!IS_ERR(virt_dev)) rknpu_dev->genpd_dev_npu1 = virt_dev; - virt_dev = dev_pm_domain_attach_by_name(dev, "npu2"); - if (!IS_ERR(virt_dev)) - rknpu_dev->genpd_dev_npu2 = virt_dev; + if (config->num_irqs > 2) { + virt_dev = dev_pm_domain_attach_by_name(dev, "npu2"); + if (!IS_ERR(virt_dev)) + rknpu_dev->genpd_dev_npu2 = virt_dev; + } rknpu_dev->multiple_domains = true; } @@ -1819,7 +1475,8 @@ static int rknpu_probe(struct platform_device *pdev) INIT_DEFERRABLE_WORK(&rknpu_dev->power_off_work, rknpu_power_off_delay_work); - if (IS_ENABLED(CONFIG_ROCKCHIP_RKNPU_SRAM) && rknpu_dev->iommu_en) { + if (IS_ENABLED(CONFIG_NO_GKI) && + IS_ENABLED(CONFIG_ROCKCHIP_RKNPU_SRAM) && rknpu_dev->iommu_en) { if (!rknpu_find_sram_resource(rknpu_dev)) { ret = rknpu_mm_create(rknpu_dev->sram_size, PAGE_SIZE, &rknpu_dev->sram_mm); @@ -1830,9 +1487,20 @@ static int rknpu_probe(struct platform_device *pdev) } } + if (IS_ENABLED(CONFIG_NO_GKI) && rknpu_dev->iommu_en && + rknpu_dev->config->nbuf_size > 0) { + rknpu_find_nbuf_resource(rknpu_dev); + if (rknpu_dev->config->cache_sgt_init != NULL) + rknpu_dev->config->cache_sgt_init(rknpu_dev); + } + + if (rknpu_dev->iommu_en) + rknpu_iommu_init_domain(rknpu_dev); + rknpu_power_off(rknpu_dev); atomic_set(&rknpu_dev->power_refcount, 0); atomic_set(&rknpu_dev->cmdline_power_refcount, 0); + atomic_set(&rknpu_dev->iommu_domain_refcount, 0); rknpu_debugger_init(rknpu_dev); rknpu_init_timer(rknpu_dev); @@ -1866,17 +1534,32 @@ static int rknpu_remove(struct platform_device *pdev) cancel_delayed_work_sync(&rknpu_dev->power_off_work); destroy_workqueue(rknpu_dev->power_off_wq); - if (IS_ENABLED(CONFIG_ROCKCHIP_RKNPU_SRAM) && rknpu_dev->sram_mm) - rknpu_mm_destroy(rknpu_dev->sram_mm); - rknpu_debugger_remove(rknpu_dev); rknpu_cancel_timer(rknpu_dev); + if (rknpu_dev->config->cache_sgt_init != NULL) { + for (i = 0; i < RKNPU_CACHE_SG_TABLE_NUM; i++) { + if (rknpu_dev->cache_sgt[i]) { + sg_free_table(rknpu_dev->cache_sgt[i]); + kfree(rknpu_dev->cache_sgt[i]); + rknpu_dev->cache_sgt[i] = NULL; + } + } + } + for (i = 0; i < rknpu_dev->config->num_irqs; i++) { WARN_ON(rknpu_dev->subcore_datas[i].job); WARN_ON(!list_empty(&rknpu_dev->subcore_datas[i].todo_list)); } + if (IS_ENABLED(CONFIG_ROCKCHIP_RKNPU_SRAM) && rknpu_dev->sram_mm) + rknpu_mm_destroy(rknpu_dev->sram_mm); + + if (rknpu_dev->iommu_en) { + rknpu_iommu_free_domains(rknpu_dev); + iommu_group_put(rknpu_dev->iommu_group); + } + #ifdef CONFIG_ROCKCHIP_RKNPU_DRM_GEM rknpu_drm_remove(rknpu_dev); #endif @@ -1908,57 +1591,41 @@ static int rknpu_remove(struct platform_device *pdev) } #ifndef FPGA_PLATFORM -#if KERNEL_VERSION(5, 10, 0) <= LINUX_VERSION_CODE -static int rknpu_runtime_suspend(struct device *dev) +#ifdef CONFIG_PM_SLEEP +static int rknpu_suspend(struct device *dev) { struct rknpu_device *rknpu_dev = dev_get_drvdata(dev); - struct rockchip_opp_info *opp_info = &rknpu_dev->opp_info; - if (opp_info->scmi_clk) { - if (clk_set_rate(opp_info->scmi_clk, POWER_DOWN_FREQ)) - LOG_DEV_ERROR(dev, "failed to restore clk rate\n"); - } - opp_info->current_rm = UINT_MAX; + rknpu_power_get(rknpu_dev); - return 0; + return pm_runtime_force_suspend(dev); +} + +static int rknpu_resume(struct device *dev) +{ + struct rknpu_device *rknpu_dev = dev_get_drvdata(dev); + + rknpu_power_put_delay(rknpu_dev); + + return pm_runtime_force_resume(dev); +} +#endif + +static int rknpu_runtime_suspend(struct device *dev) +{ + return rknpu_devfreq_runtime_suspend(dev); } static int rknpu_runtime_resume(struct device *dev) { - struct rknpu_device *rknpu_dev = dev_get_drvdata(dev); - struct rockchip_opp_info *opp_info = &rknpu_dev->opp_info; - int ret = 0; - - if (!rknpu_dev->current_freq || !rknpu_dev->current_volt) - return 0; - - ret = clk_bulk_prepare_enable(opp_info->num_clks, opp_info->clks); - if (ret) { - LOG_DEV_ERROR(dev, "failed to enable opp clks\n"); - return ret; - } - - if (opp_info->data && opp_info->data->set_read_margin) - opp_info->data->set_read_margin(dev, opp_info, - opp_info->target_rm); - if (opp_info->scmi_clk) { - if (clk_set_rate(opp_info->scmi_clk, rknpu_dev->current_freq)) - LOG_DEV_ERROR(dev, "failed to set power down rate\n"); - } - - clk_bulk_disable_unprepare(opp_info->num_clks, opp_info->clks); - - return ret; + return rknpu_devfreq_runtime_resume(dev); } static const struct dev_pm_ops rknpu_pm_ops = { - SET_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend, - pm_runtime_force_resume) - SET_RUNTIME_PM_OPS(rknpu_runtime_suspend, rknpu_runtime_resume, - NULL) + SET_SYSTEM_SLEEP_PM_OPS(rknpu_suspend, rknpu_resume) SET_RUNTIME_PM_OPS( + rknpu_runtime_suspend, rknpu_runtime_resume, NULL) }; #endif -#endif static struct platform_driver rknpu_driver = { .probe = rknpu_probe, @@ -1967,9 +1634,7 @@ static struct platform_driver rknpu_driver = { .owner = THIS_MODULE, .name = "RKNPU", #ifndef FPGA_PLATFORM -#if KERNEL_VERSION(5, 5, 0) < LINUX_VERSION_CODE .pm = &rknpu_pm_ops, -#endif #endif .of_match_table = of_match_ptr(rknpu_of_match), }, @@ -1994,3 +1659,6 @@ MODULE_ALIAS("rockchip-rknpu"); MODULE_LICENSE("GPL v2"); MODULE_VERSION(RKNPU_GET_DRV_VERSION_STRING(DRIVER_MAJOR, DRIVER_MINOR, DRIVER_PATCHLEVEL)); +#if KERNEL_VERSION(5, 16, 0) < LINUX_VERSION_CODE +MODULE_IMPORT_NS(DMA_BUF); +#endif diff --git a/target/linux/rockchip/rk35xx/files/drivers/rknpu/rknpu_gem.c b/target/linux/rockchip/rk35xx/files/drivers/rknpu/rknpu_gem.c index 38d3197fad..a5c5354ed8 100644 --- a/target/linux/rockchip/rk35xx/files/drivers/rknpu/rknpu_gem.c +++ b/target/linux/rockchip/rk35xx/files/drivers/rknpu/rknpu_gem.c @@ -10,12 +10,13 @@ #include #include +#include #include #include #include -#include #include #include +#include #include #if KERNEL_VERSION(5, 10, 0) <= LINUX_VERSION_CODE @@ -25,6 +26,7 @@ #include "rknpu_drv.h" #include "rknpu_ioctl.h" #include "rknpu_gem.h" +#include "rknpu_iommu.h" #define RKNPU_GEM_ALLOC_FROM_PAGES 1 @@ -36,6 +38,8 @@ static int rknpu_gem_get_pages(struct rknpu_gem_object *rknpu_obj) dma_addr_t dma_addr = 0; dma_addr_t phys = 0; int ret = -EINVAL, i = 0; + bool iova_aligned = + !(rknpu_obj->flags & RKNPU_MEM_IOMMU_LIMIT_IOVA_ALIGNMENT); rknpu_obj->pages = drm_gem_get_pages(&rknpu_obj->base); if (IS_ERR(rknpu_obj->pages)) { @@ -59,14 +63,16 @@ static int rknpu_gem_get_pages(struct rknpu_gem_object *rknpu_obj) goto put_pages; } - ret = dma_map_sg(drm->dev, rknpu_obj->sgt->sgl, rknpu_obj->sgt->nents, - DMA_BIDIRECTIONAL); + ret = rknpu_iommu_dma_map_sg(drm->dev, rknpu_obj->sgt->sgl, + rknpu_obj->sgt->nents, DMA_BIDIRECTIONAL, + iova_aligned); if (ret == 0) { ret = -EFAULT; LOG_DEV_ERROR(drm->dev, "%s: dma map %zu fail\n", __func__, rknpu_obj->size); goto free_sgt; } + iommu_flush_iotlb_all(iommu_get_domain_for_dev(drm->dev)); if (rknpu_obj->flags & RKNPU_MEM_KERNEL_MAPPING) { rknpu_obj->cookie = vmap(rknpu_obj->pages, rknpu_obj->num_pages, @@ -93,8 +99,9 @@ static int rknpu_gem_get_pages(struct rknpu_gem_object *rknpu_obj) return 0; unmap_sg: - dma_unmap_sg(drm->dev, rknpu_obj->sgt->sgl, rknpu_obj->sgt->nents, - DMA_BIDIRECTIONAL); + rknpu_iommu_dma_unmap_sg(drm->dev, rknpu_obj->sgt->sgl, + rknpu_obj->sgt->nents, DMA_BIDIRECTIONAL, + iova_aligned); free_sgt: sg_free_table(rknpu_obj->sgt); @@ -109,6 +116,8 @@ put_pages: static void rknpu_gem_put_pages(struct rknpu_gem_object *rknpu_obj) { struct drm_device *drm = rknpu_obj->base.dev; + bool iova_aligned = + !(rknpu_obj->flags & RKNPU_MEM_IOMMU_LIMIT_IOVA_ALIGNMENT); if (rknpu_obj->flags & RKNPU_MEM_KERNEL_MAPPING) { vunmap(rknpu_obj->kv_addr); @@ -116,8 +125,9 @@ static void rknpu_gem_put_pages(struct rknpu_gem_object *rknpu_obj) } if (rknpu_obj->sgt != NULL) { - dma_unmap_sg(drm->dev, rknpu_obj->sgt->sgl, - rknpu_obj->sgt->nents, DMA_BIDIRECTIONAL); + rknpu_iommu_dma_unmap_sg(drm->dev, rknpu_obj->sgt->sgl, + rknpu_obj->sgt->nents, + DMA_BIDIRECTIONAL, iova_aligned); sg_free_table(rknpu_obj->sgt); kfree(rknpu_obj->sgt); } @@ -181,7 +191,9 @@ static int rknpu_gem_alloc_buf(struct rknpu_gem_object *rknpu_obj) if (rknpu_obj->flags & RKNPU_MEM_ZEROING) gfp_mask |= __GFP_ZERO; - if (!(rknpu_obj->flags & RKNPU_MEM_NON_DMA32)) { + if (!rknpu_dev->iommu_en || + rknpu_dev->config->dma_mask <= DMA_BIT_MASK(32) || + (rknpu_obj->flags & RKNPU_MEM_DMA32)) { gfp_mask &= ~__GFP_HIGHMEM; gfp_mask |= __GFP_DMA32; } @@ -194,9 +206,9 @@ static int rknpu_gem_alloc_buf(struct rknpu_gem_object *rknpu_obj) return -ENOMEM; } - rknpu_obj->cookie = - dma_alloc_attrs(drm->dev, rknpu_obj->size, &rknpu_obj->dma_addr, - gfp_mask, rknpu_obj->dma_attrs); + rknpu_obj->cookie = dma_alloc_attrs(drm->dev, rknpu_obj->size, + &rknpu_obj->dma_addr, gfp_mask, + rknpu_obj->dma_attrs); if (!rknpu_obj->cookie) { /* * when RKNPU_MEM_CONTIGUOUS and IOMMU is available @@ -210,10 +222,9 @@ static int rknpu_gem_alloc_buf(struct rknpu_gem_object *rknpu_obj) rknpu_obj->size); rknpu_obj->dma_attrs &= ~DMA_ATTR_FORCE_CONTIGUOUS; rknpu_obj->flags |= RKNPU_MEM_NON_CONTIGUOUS; - rknpu_obj->cookie = - dma_alloc_attrs(drm->dev, rknpu_obj->size, - &rknpu_obj->dma_addr, gfp_mask, - rknpu_obj->dma_attrs); + rknpu_obj->cookie = dma_alloc_attrs( + drm->dev, rknpu_obj->size, &rknpu_obj->dma_addr, + gfp_mask, rknpu_obj->dma_attrs); if (!rknpu_obj->cookie) { LOG_DEV_ERROR( drm->dev, @@ -252,10 +263,15 @@ static int rknpu_gem_alloc_buf(struct rknpu_gem_object *rknpu_obj) i, &s->dma_address, s->length); } - if (drm_prime_sg_to_page_addr_arrays(sgt, rknpu_obj->pages, NULL, - nr_pages)) { - LOG_DEV_ERROR(drm->dev, "invalid sgtable.\n"); - ret = -EINVAL; +#if KERNEL_VERSION(6, 1, 0) > LINUX_VERSION_CODE + ret = drm_prime_sg_to_page_addr_arrays(sgt, rknpu_obj->pages, NULL, + nr_pages); +#else + ret = drm_prime_sg_to_page_array(sgt, rknpu_obj->pages, nr_pages); +#endif + + if (ret < 0) { + LOG_DEV_ERROR(drm->dev, "invalid sgtable, ret: %d\n", ret); goto err_free_sg_table; } @@ -334,9 +350,28 @@ static int rknpu_gem_handle_destroy(struct drm_file *file_priv, return drm_gem_handle_delete(file_priv, handle); } +#if KERNEL_VERSION(6, 1, 0) <= LINUX_VERSION_CODE +static const struct vm_operations_struct vm_ops = { + .fault = rknpu_gem_fault, + .open = drm_gem_vm_open, + .close = drm_gem_vm_close, +}; + +static const struct drm_gem_object_funcs rknpu_gem_object_funcs = { + .free = rknpu_gem_free_object, + .export = drm_gem_prime_export, + .get_sg_table = rknpu_gem_prime_get_sg_table, + .vmap = rknpu_gem_prime_vmap, + .vunmap = rknpu_gem_prime_vunmap, + .mmap = rknpu_gem_mmap_obj, + .vm_ops = &vm_ops, +}; +#endif + static struct rknpu_gem_object *rknpu_gem_init(struct drm_device *drm, unsigned long size) { + struct rknpu_device *rknpu_dev = drm->dev_private; struct rknpu_gem_object *rknpu_obj = NULL; struct drm_gem_object *obj = NULL; gfp_t gfp_mask; @@ -347,6 +382,9 @@ static struct rknpu_gem_object *rknpu_gem_init(struct drm_device *drm, return ERR_PTR(-ENOMEM); obj = &rknpu_obj->base; +#if KERNEL_VERSION(6, 1, 0) <= LINUX_VERSION_CODE + obj->funcs = &rknpu_gem_object_funcs; +#endif ret = drm_gem_object_init(drm, obj, size); if (ret < 0) { @@ -362,7 +400,9 @@ static struct rknpu_gem_object *rknpu_gem_init(struct drm_device *drm, if (rknpu_obj->flags & RKNPU_MEM_ZEROING) gfp_mask |= __GFP_ZERO; - if (!(rknpu_obj->flags & RKNPU_MEM_NON_DMA32)) { + if (!rknpu_dev->iommu_en || + rknpu_dev->config->dma_mask <= DMA_BIT_MASK(32) || + (rknpu_obj->flags & RKNPU_MEM_DMA32)) { gfp_mask &= ~__GFP_HIGHMEM; gfp_mask |= __GFP_DMA32; } @@ -379,7 +419,52 @@ static void rknpu_gem_release(struct rknpu_gem_object *rknpu_obj) kfree(rknpu_obj); } -static int rknpu_gem_alloc_buf_with_sram(struct rknpu_gem_object *rknpu_obj) +static int rknpu_iommu_map_with_cache_sgt(struct iommu_domain *domain, + struct rknpu_device *rknpu_dev, + struct rknpu_gem_object *rknpu_obj, + unsigned long cache_size) +{ + phys_addr_t cache_start = 0; + unsigned long iova_start = rknpu_obj->iova_start; + struct scatterlist *s = NULL; + unsigned long length = cache_size; + unsigned long size = 0; + int i = 0; + int ret = 0; + int index = 0; + + switch (rknpu_obj->core_mask) { + case RKNPU_CORE0_MASK: + index = 0; + break; + case RKNPU_CORE1_MASK: + index = 1; + break; + default: + break; + } + + for_each_sgtable_sg(rknpu_dev->cache_sgt[index], s, i) { + cache_start = rknpu_dev->nbuf_start + s->offset; + size = length < s->length ? length : s->length; + ret = iommu_map(domain, iova_start, cache_start, size, + IOMMU_READ | IOMMU_WRITE); + if (ret) { + LOG_ERROR("cache iommu_map error: %d\n", ret); + return ret; + } + length -= size; + iova_start += size; + + if (length == 0) + break; + } + + return ret; +} + +static int rknpu_gem_alloc_buf_with_cache(struct rknpu_gem_object *rknpu_obj, + enum rknpu_cache_type cache_type) { struct drm_device *drm = rknpu_obj->base.dev; struct rknpu_device *rknpu_dev = drm->dev_private; @@ -392,20 +477,42 @@ static int rknpu_gem_alloc_buf_with_sram(struct rknpu_gem_object *rknpu_obj) unsigned long offset = 0; int i = 0; int ret = -EINVAL; + phys_addr_t cache_start = 0; + unsigned long cache_offset = 0; + unsigned long cache_size = 0; + bool iova_aligned = + !(rknpu_obj->flags & RKNPU_MEM_IOMMU_LIMIT_IOVA_ALIGNMENT); - /* iova map to sram */ + switch (cache_type) { + case RKNPU_CACHE_SRAM: + cache_start = rknpu_dev->sram_start; + cache_offset = rknpu_obj->sram_obj->range_start * + rknpu_dev->sram_mm->chunk_size; + cache_size = rknpu_obj->sram_size; + break; + case RKNPU_CACHE_NBUF: + cache_start = rknpu_dev->nbuf_start; + cache_offset = 0; + cache_size = rknpu_obj->nbuf_size; + break; + default: + LOG_ERROR("Unknown rknpu_cache_type: %d", cache_type); + return -EINVAL; + } + + /* iova map to cache */ domain = iommu_get_domain_for_dev(rknpu_dev->dev); if (!domain) { LOG_ERROR("failed to get iommu domain!"); return -EINVAL; } - cookie = domain->iova_cookie; + cookie = (void *)domain->iova_cookie; iovad = &cookie->iovad; - rknpu_obj->iova_size = - iova_align(iovad, rknpu_obj->sram_size + rknpu_obj->size); + rknpu_obj->iova_size = iova_align(iovad, cache_size + rknpu_obj->size); rknpu_obj->iova_start = rknpu_iommu_dma_alloc_iova( - domain, rknpu_obj->iova_size, dma_get_mask(drm->dev), drm->dev); + domain, rknpu_obj->iova_size, dma_get_mask(drm->dev), drm->dev, + iova_aligned); if (!rknpu_obj->iova_start) { LOG_ERROR("iommu_dma_alloc_iova failed\n"); return -ENOMEM; @@ -415,20 +522,20 @@ static int rknpu_gem_alloc_buf_with_sram(struct rknpu_gem_object *rknpu_obj) &rknpu_obj->iova_start, rknpu_obj->iova_size); /* - * Overview SRAM + DDR map to IOVA + * Overview cache + DDR map to IOVA * -------- - * sram_size: rknpu_obj->sram_size - * - allocate from SRAM, this size value has been page-aligned + * cache_size: + * - allocate from CACHE, this size value has been page-aligned * size: rknpu_obj->size * - allocate from DDR pages, this size value has been page-aligned * iova_size: rknpu_obj->iova_size - * - from iova_align(sram_size + size) - * - it may be larger than the (sram_size + size), and the larger part is not mapped + * - from iova_align(cache_size + size) + * - it may be larger than the (cache_size + size), and the larger part is not mapped * -------- * - * |<- sram_size ->| |<- - - - size - - - ->| + * |<- cache_size ->| |<- - - - size - - - ->| * +---------------+ +----------------------+ - * | SRAM | | DDR | + * | CACHE | | DDR | * +---------------+ +----------------------+ * | | * | V | V | @@ -438,20 +545,23 @@ static int rknpu_gem_alloc_buf_with_sram(struct rknpu_gem_object *rknpu_obj) * |<- - - - - - - iova_size - - - - - - ->| * */ - offset = rknpu_obj->sram_obj->range_start * - rknpu_dev->sram_mm->chunk_size; - ret = iommu_map(domain, rknpu_obj->iova_start, - rknpu_dev->sram_start + offset, rknpu_obj->sram_size, - IOMMU_READ | IOMMU_WRITE); + if (!rknpu_obj->cache_with_sgt) + ret = iommu_map(domain, rknpu_obj->iova_start, + cache_start + cache_offset, cache_size, + IOMMU_READ | IOMMU_WRITE); + else + ret = rknpu_iommu_map_with_cache_sgt(domain, rknpu_dev, + rknpu_obj, cache_size); + if (ret) { - LOG_ERROR("sram iommu_map error: %d\n", ret); + LOG_ERROR("cache iommu_map error: %d\n", ret); goto free_iova; } rknpu_obj->dma_addr = rknpu_obj->iova_start; if (rknpu_obj->size == 0) { - LOG_INFO("allocate sram size: %lu\n", rknpu_obj->sram_size); + LOG_INFO("allocate cache size: %lu\n", cache_size); return 0; } @@ -459,7 +569,7 @@ static int rknpu_gem_alloc_buf_with_sram(struct rknpu_gem_object *rknpu_obj) if (IS_ERR(rknpu_obj->pages)) { ret = PTR_ERR(rknpu_obj->pages); LOG_ERROR("failed to get pages: %d\n", ret); - goto sram_unmap; + goto cache_unmap; } rknpu_obj->num_pages = rknpu_obj->size >> PAGE_SHIFT; @@ -478,7 +588,7 @@ static int rknpu_gem_alloc_buf_with_sram(struct rknpu_gem_object *rknpu_obj) } length = rknpu_obj->size; - offset = rknpu_obj->iova_start + rknpu_obj->sram_size; + offset = rknpu_obj->iova_start + cache_size; for_each_sg(rknpu_obj->sgt->sgl, s, rknpu_obj->sgt->nents, i) { size = (length < s->length) ? length : s->length; @@ -497,13 +607,13 @@ static int rknpu_gem_alloc_buf_with_sram(struct rknpu_gem_object *rknpu_obj) break; } - LOG_INFO("allocate size: %lu with sram size: %lu\n", rknpu_obj->size, - rknpu_obj->sram_size); + LOG_INFO("allocate size: %lu with cache size: %lu\n", rknpu_obj->size, + cache_size); return 0; sgl_unmap: - iommu_unmap(domain, rknpu_obj->iova_start + rknpu_obj->sram_size, + iommu_unmap(domain, rknpu_obj->iova_start + cache_size, rknpu_obj->size - length); sg_free_table(rknpu_obj->sgt); kfree(rknpu_obj->sgt); @@ -511,34 +621,48 @@ sgl_unmap: put_pages: drm_gem_put_pages(&rknpu_obj->base, rknpu_obj->pages, false, false); -sram_unmap: - iommu_unmap(domain, rknpu_obj->iova_start, rknpu_obj->sram_size); +cache_unmap: + iommu_unmap(domain, rknpu_obj->iova_start, cache_size); free_iova: - rknpu_iommu_dma_free_iova(domain->iova_cookie, rknpu_obj->iova_start, - rknpu_obj->iova_size); + rknpu_iommu_dma_free_iova((void *)domain->iova_cookie, + rknpu_obj->iova_start, rknpu_obj->iova_size, + iova_aligned); return ret; } -static void rknpu_gem_free_buf_with_sram(struct rknpu_gem_object *rknpu_obj) +static void rknpu_gem_free_buf_with_cache(struct rknpu_gem_object *rknpu_obj, + enum rknpu_cache_type cache_type) { struct drm_device *drm = rknpu_obj->base.dev; struct rknpu_device *rknpu_dev = drm->dev_private; struct iommu_domain *domain = NULL; + unsigned long cache_size = 0; + bool iova_aligned = + !(rknpu_obj->flags & RKNPU_MEM_IOMMU_LIMIT_IOVA_ALIGNMENT); + + switch (cache_type) { + case RKNPU_CACHE_SRAM: + cache_size = rknpu_obj->sram_size; + break; + case RKNPU_CACHE_NBUF: + cache_size = rknpu_obj->nbuf_size; + break; + default: + LOG_ERROR("Unknown rknpu_cache_type: %d", cache_type); + return; + } domain = iommu_get_domain_for_dev(rknpu_dev->dev); if (domain) { - iommu_unmap(domain, rknpu_obj->iova_start, - rknpu_obj->sram_size); + iommu_unmap(domain, rknpu_obj->iova_start, cache_size); if (rknpu_obj->size > 0) - iommu_unmap(domain, - rknpu_obj->iova_start + - rknpu_obj->sram_size, + iommu_unmap(domain, rknpu_obj->iova_start + cache_size, rknpu_obj->size); - rknpu_iommu_dma_free_iova(domain->iova_cookie, + rknpu_iommu_dma_free_iova((void *)domain->iova_cookie, rknpu_obj->iova_start, - rknpu_obj->iova_size); + rknpu_obj->iova_size, iova_aligned); } if (rknpu_obj->pages) @@ -551,10 +675,10 @@ static void rknpu_gem_free_buf_with_sram(struct rknpu_gem_object *rknpu_obj) } } -struct rknpu_gem_object *rknpu_gem_object_create(struct drm_device *drm, - unsigned int flags, - unsigned long size, - unsigned long sram_size) +struct rknpu_gem_object * +rknpu_gem_object_create(struct drm_device *drm, unsigned int flags, + unsigned long size, unsigned long sram_size, + int iommu_domain_id, unsigned int core_mask) { struct rknpu_device *rknpu_dev = drm->dev_private; struct rknpu_gem_object *rknpu_obj = NULL; @@ -568,6 +692,18 @@ struct rknpu_gem_object *rknpu_gem_object_create(struct drm_device *drm, remain_ddr_size = round_up(size, PAGE_SIZE); + rknpu_obj = rknpu_gem_init(drm, remain_ddr_size); + if (IS_ERR(rknpu_obj)) + return rknpu_obj; + + if (rknpu_iommu_domain_get_and_switch(rknpu_dev, iommu_domain_id)) { + LOG_DEV_ERROR(rknpu_dev->dev, "%s error\n", __func__); + rknpu_gem_release(rknpu_obj); + return ERR_PTR(-EINVAL); + } + + rknpu_obj->iommu_domain_id = iommu_domain_id; + if (!rknpu_dev->iommu_en && (flags & RKNPU_MEM_NON_CONTIGUOUS)) { /* * when no IOMMU is available, all allocated buffers are @@ -578,6 +714,9 @@ struct rknpu_gem_object *rknpu_gem_object_create(struct drm_device *drm, "non-contiguous allocation is not supported without IOMMU, falling back to contiguous buffer\n"); } + /* set memory type and cache attribute from user side. */ + rknpu_obj->flags = flags; + if (IS_ENABLED(CONFIG_ROCKCHIP_RKNPU_SRAM) && (flags & RKNPU_MEM_TRY_ALLOC_SRAM) && rknpu_dev->sram_size > 0) { size_t sram_free_size = 0; @@ -586,12 +725,7 @@ struct rknpu_gem_object *rknpu_gem_object_create(struct drm_device *drm, if (sram_size != 0) sram_size = round_up(sram_size, PAGE_SIZE); - rknpu_obj = rknpu_gem_init(drm, remain_ddr_size); - if (IS_ERR(rknpu_obj)) - return rknpu_obj; - - /* set memory type and cache attribute from user side. */ - rknpu_obj->flags = flags; + rknpu_obj->cache_with_sgt = 0; sram_free_size = rknpu_dev->sram_mm->free_chunks * rknpu_dev->sram_mm->chunk_size; @@ -617,32 +751,57 @@ struct rknpu_gem_object *rknpu_gem_object_create(struct drm_device *drm, if (real_sram_size > 0) { rknpu_obj->sram_size = real_sram_size; - ret = rknpu_gem_alloc_buf_with_sram(rknpu_obj); + ret = rknpu_gem_alloc_buf_with_cache(rknpu_obj, + RKNPU_CACHE_SRAM); if (ret < 0) goto mm_free; remain_ddr_size = 0; } + } else if (IS_ENABLED(CONFIG_NO_GKI) && + (flags & RKNPU_MEM_TRY_ALLOC_NBUF) && + rknpu_dev->nbuf_size > 0) { + size_t nbuf_size = rknpu_dev->nbuf_size; + + rknpu_obj->cache_with_sgt = 0; + + if (core_mask == RKNPU_CORE_AUTO_MASK || + core_mask == RKNPU_CORE0_MASK || + core_mask == RKNPU_CORE1_MASK) { + if (rknpu_dev->cache_sgt[0]) + rknpu_obj->cache_with_sgt = 1; + nbuf_size = rknpu_dev->nbuf_size / + rknpu_dev->config->num_irqs; + } + + rknpu_obj->core_mask = core_mask; + nbuf_size = remain_ddr_size <= nbuf_size ? remain_ddr_size : + nbuf_size; + + if (nbuf_size > 0) { + rknpu_obj->nbuf_size = nbuf_size; + + ret = rknpu_gem_alloc_buf_with_cache(rknpu_obj, + RKNPU_CACHE_NBUF); + if (ret < 0) + goto gem_release; + remain_ddr_size = 0; + } } if (remain_ddr_size > 0) { - rknpu_obj = rknpu_gem_init(drm, remain_ddr_size); - if (IS_ERR(rknpu_obj)) - return rknpu_obj; - - /* set memory type and cache attribute from user side. */ - rknpu_obj->flags = flags; - ret = rknpu_gem_alloc_buf(rknpu_obj); if (ret < 0) goto gem_release; } - if (rknpu_obj) - LOG_DEBUG( - "created dma addr: %pad, cookie: %p, ddr size: %lu, sram size: %lu, attrs: %#lx, flags: %#x\n", - &rknpu_obj->dma_addr, rknpu_obj->cookie, - rknpu_obj->size, rknpu_obj->sram_size, - rknpu_obj->dma_attrs, rknpu_obj->flags); + rknpu_iommu_domain_put(rknpu_dev); + + LOG_DEBUG( + "created dma addr: %pad, cookie: %p, ddr size: %lu, sram size: %lu, nbuf size: %lu, attrs: %#lx, flags: %#x, iommu domain id: %d\n", + &rknpu_obj->dma_addr, rknpu_obj->cookie, rknpu_obj->size, + rknpu_obj->sram_size, rknpu_obj->nbuf_size, + rknpu_obj->dma_attrs, rknpu_obj->flags, + rknpu_obj->iommu_domain_id); return rknpu_obj; @@ -654,18 +813,36 @@ mm_free: gem_release: rknpu_gem_release(rknpu_obj); + rknpu_iommu_domain_put(rknpu_dev); + return ERR_PTR(ret); } void rknpu_gem_object_destroy(struct rknpu_gem_object *rknpu_obj) { struct drm_gem_object *obj = &rknpu_obj->base; + struct rknpu_device *rknpu_dev = obj->dev->dev_private; + int wait_count = 0; + int ret = -EINVAL; LOG_DEBUG( "destroy dma addr: %pad, cookie: %p, size: %lu, attrs: %#lx, flags: %#x, handle count: %d\n", &rknpu_obj->dma_addr, rknpu_obj->cookie, rknpu_obj->size, rknpu_obj->dma_attrs, rknpu_obj->flags, obj->handle_count); + do { + ret = rknpu_iommu_domain_get_and_switch( + rknpu_dev, rknpu_obj->iommu_domain_id); + + if (ret && ++wait_count >= 3) { + LOG_DEV_ERROR( + rknpu_dev->dev, + "failed to destroy dma addr: %pad, size: %lu\n", + &rknpu_obj->dma_addr, rknpu_obj->size); + return; + } + } while (ret); + /* * do not release memory region from exporter. * @@ -678,21 +855,25 @@ void rknpu_gem_object_destroy(struct rknpu_gem_object *rknpu_obj) } else { if (IS_ENABLED(CONFIG_ROCKCHIP_RKNPU_SRAM) && rknpu_obj->sram_size > 0) { - struct rknpu_device *rknpu_dev = obj->dev->dev_private; - if (rknpu_obj->sram_obj != NULL) rknpu_mm_free(rknpu_dev->sram_mm, rknpu_obj->sram_obj); - rknpu_gem_free_buf_with_sram(rknpu_obj); + rknpu_gem_free_buf_with_cache(rknpu_obj, + RKNPU_CACHE_SRAM); + } else if (IS_ENABLED(CONFIG_NO_GKI) && + rknpu_obj->nbuf_size > 0) { + rknpu_gem_free_buf_with_cache(rknpu_obj, + RKNPU_CACHE_NBUF); } else { rknpu_gem_free_buf(rknpu_obj); } } rknpu_gem_release(rknpu_obj); + rknpu_iommu_domain_put(rknpu_dev); } -int rknpu_gem_create_ioctl(struct drm_device *dev, void *data, +int rknpu_gem_create_ioctl(struct drm_device *drm, void *data, struct drm_file *file_priv) { struct rknpu_mem_create *args = data; @@ -701,8 +882,10 @@ int rknpu_gem_create_ioctl(struct drm_device *dev, void *data, rknpu_obj = rknpu_gem_object_find(file_priv, args->handle); if (!rknpu_obj) { - rknpu_obj = rknpu_gem_object_create( - dev, args->flags, args->size, args->sram_size); + rknpu_obj = rknpu_gem_object_create(drm, args->flags, + args->size, args->sram_size, + args->iommu_domain_id, + args->core_mask); if (IS_ERR(rknpu_obj)) return PTR_ERR(rknpu_obj); @@ -738,19 +921,35 @@ int rknpu_gem_map_ioctl(struct drm_device *dev, void *data, #endif } -int rknpu_gem_destroy_ioctl(struct drm_device *dev, void *data, +int rknpu_gem_destroy_ioctl(struct drm_device *drm, void *data, struct drm_file *file_priv) { + struct rknpu_device *rknpu_dev = drm->dev_private; struct rknpu_gem_object *rknpu_obj = NULL; struct rknpu_mem_destroy *args = data; + int ret = 0; + int wait_count = 0; rknpu_obj = rknpu_gem_object_find(file_priv, args->handle); if (!rknpu_obj) return -EINVAL; - // rknpu_gem_object_put(&rknpu_obj->base); + do { + ret = rknpu_iommu_domain_get_and_switch( + rknpu_dev, rknpu_obj->iommu_domain_id); - return rknpu_gem_handle_destroy(file_priv, args->handle); + if (ret && ++wait_count >= 3) { + LOG_DEV_ERROR(rknpu_dev->dev, + "failed to destroy memory\n"); + return ret; + } + } while (ret); + + ret = rknpu_gem_handle_destroy(file_priv, args->handle); + + rknpu_iommu_domain_put(rknpu_dev); + + return ret; } #if RKNPU_GEM_ALLOC_FROM_PAGES @@ -796,7 +995,7 @@ static int rknpu_gem_mmap_pages(struct rknpu_gem_object *rknpu_obj, struct drm_device *drm = rknpu_obj->base.dev; int ret = -EINVAL; - vma->vm_flags |= VM_MIXEDMAP; + vm_flags_set(vma, VM_MIXEDMAP); ret = __vm_map_pages(vma, rknpu_obj->pages, rknpu_obj->num_pages, vma->vm_pgoff); @@ -808,6 +1007,128 @@ static int rknpu_gem_mmap_pages(struct rknpu_gem_object *rknpu_obj, } #endif +static int rknpu_remap_pfn_with_cache_sgt(struct rknpu_device *rknpu_dev, + struct rknpu_gem_object *rknpu_obj, + struct vm_area_struct *vma, + unsigned long cache_size) +{ + phys_addr_t cache_start = 0; + unsigned long vm_start = vma->vm_start; + struct scatterlist *s = NULL; + unsigned long length = cache_size; + unsigned long size = 0; + int i = 0; + int ret = 0; + int index = 0; + + switch (rknpu_obj->core_mask) { + case RKNPU_CORE0_MASK: + index = 0; + break; + case RKNPU_CORE1_MASK: + index = 1; + break; + default: + break; + } + + for_each_sgtable_sg(rknpu_dev->cache_sgt[index], s, i) { + cache_start = rknpu_dev->nbuf_start + s->offset; + size = length < s->length ? length : s->length; + + vma->vm_pgoff = __phys_to_pfn(cache_start); + ret = remap_pfn_range(vma, vm_start, vma->vm_pgoff, size, + vma->vm_page_prot); + + if (ret) { + LOG_ERROR("cache remap_pfn_range error: %d\n", ret); + return ret; + } + length -= size; + vm_start += size; + + if (length == 0) + break; + } + + return ret; +} + +static int rknpu_gem_mmap_cache(struct rknpu_gem_object *rknpu_obj, + struct vm_area_struct *vma, + enum rknpu_cache_type cache_type) +{ + struct drm_device *drm = rknpu_obj->base.dev; +#if RKNPU_GEM_ALLOC_FROM_PAGES + struct rknpu_device *rknpu_dev = drm->dev_private; +#endif + unsigned long vm_size = 0; + int ret = -EINVAL; + unsigned long offset = 0; + unsigned long num_pages = 0; + int i = 0; + phys_addr_t cache_start = 0; + unsigned long cache_offset = 0; + unsigned long cache_size = 0; + + switch (cache_type) { + case RKNPU_CACHE_SRAM: + cache_start = rknpu_dev->sram_start; + cache_offset = rknpu_obj->sram_obj->range_start * + rknpu_dev->sram_mm->chunk_size; + cache_size = rknpu_obj->sram_size; + break; + case RKNPU_CACHE_NBUF: + cache_start = rknpu_dev->nbuf_start; + cache_offset = 0; + cache_size = rknpu_obj->nbuf_size; + break; + default: + LOG_ERROR("Unknown rknpu_cache_type: %d", cache_type); + return -EINVAL; + } + + vm_flags_set(vma, VM_MIXEDMAP); + + vm_size = vma->vm_end - vma->vm_start; + + /* + * Convert a physical address in a cache area to a page frame number (PFN), + * and store the resulting PFN in the vm_pgoff field of the given VMA. + * + * NOTE: This conversion carries a risk because the resulting PFN is not a true + * page frame number and may not be valid or usable in all contexts. + */ + + if (!rknpu_obj->cache_with_sgt) { + vma->vm_pgoff = __phys_to_pfn(cache_start + cache_offset); + + ret = remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff, + cache_size, vma->vm_page_prot); + } else + ret = rknpu_remap_pfn_with_cache_sgt(rknpu_dev, rknpu_obj, vma, + cache_size); + + if (ret) + return -EAGAIN; + + if (rknpu_obj->size == 0) + return 0; + + offset = cache_size; + + num_pages = (vm_size - cache_size) / PAGE_SIZE; + for (i = 0; i < num_pages; ++i) { + ret = vm_insert_page(vma, vma->vm_start + offset, + rknpu_obj->pages[i]); + if (ret < 0) + return ret; + offset += PAGE_SIZE; + } + + return 0; +} + static int rknpu_gem_mmap_buffer(struct rknpu_gem_object *rknpu_obj, struct vm_area_struct *vma) { @@ -823,7 +1144,8 @@ static int rknpu_gem_mmap_buffer(struct rknpu_gem_object *rknpu_obj, * vm_pgoff (used as a fake buffer offset by DRM) to 0 as we want to map * the whole buffer. */ - vma->vm_flags &= ~VM_PFNMAP; + vm_flags_set(vma, VM_DONTCOPY | VM_DONTEXPAND | VM_DONTDUMP | VM_IO); + vm_flags_clear(vma, VM_PFNMAP); vma->vm_pgoff = 0; vm_size = vma->vm_end - vma->vm_start; @@ -832,38 +1154,10 @@ static int rknpu_gem_mmap_buffer(struct rknpu_gem_object *rknpu_obj, if (vm_size > rknpu_obj->size) return -EINVAL; - if (rknpu_obj->sram_size > 0) { - unsigned long offset = 0; - unsigned long num_pages = 0; - int i = 0; - - vma->vm_flags |= VM_MIXEDMAP; - - offset = rknpu_obj->sram_obj->range_start * - rknpu_dev->sram_mm->chunk_size; - vma->vm_pgoff = __phys_to_pfn(rknpu_dev->sram_start + offset); - - ret = remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff, - rknpu_obj->sram_size, vma->vm_page_prot); - if (ret) - return -EAGAIN; - - if (rknpu_obj->size == 0) - return 0; - - offset = rknpu_obj->sram_size; - - num_pages = (vm_size - rknpu_obj->sram_size) / PAGE_SIZE; - for (i = 0; i < num_pages; ++i) { - ret = vm_insert_page(vma, vma->vm_start + offset, - rknpu_obj->pages[i]); - if (ret < 0) - return ret; - offset += PAGE_SIZE; - } - - return 0; - } + if (rknpu_obj->sram_size > 0) + return rknpu_gem_mmap_cache(rknpu_obj, vma, RKNPU_CACHE_SRAM); + else if (rknpu_obj->nbuf_size > 0) + return rknpu_gem_mmap_cache(rknpu_obj, vma, RKNPU_CACHE_NBUF); #if RKNPU_GEM_ALLOC_FROM_PAGES if ((rknpu_obj->flags & RKNPU_MEM_NON_CONTIGUOUS) && @@ -885,7 +1179,11 @@ static int rknpu_gem_mmap_buffer(struct rknpu_gem_object *rknpu_obj, void rknpu_gem_free_object(struct drm_gem_object *obj) { + struct rknpu_device *rknpu_dev = obj->dev->dev_private; + + rknpu_power_get(rknpu_dev); rknpu_gem_object_destroy(to_rknpu_obj(obj)); + rknpu_power_put_delay(rknpu_dev); } int rknpu_gem_dumb_create(struct drm_file *file_priv, struct drm_device *drm, @@ -909,7 +1207,7 @@ int rknpu_gem_dumb_create(struct drm_file *file_priv, struct drm_device *drm, else flags = RKNPU_MEM_CONTIGUOUS | RKNPU_MEM_WRITE_COMBINE; - rknpu_obj = rknpu_gem_object_create(drm, flags, args->size, 0); + rknpu_obj = rknpu_gem_object_create(drm, flags, args->size, 0, 0, 0); if (IS_ERR(rknpu_obj)) { LOG_DEV_ERROR(drm->dev, "gem object allocate failed.\n"); return PTR_ERR(rknpu_obj); @@ -1045,8 +1343,7 @@ out: } #endif -static int rknpu_gem_mmap_obj(struct drm_gem_object *obj, - struct vm_area_struct *vma) +int rknpu_gem_mmap_obj(struct drm_gem_object *obj, struct vm_area_struct *vma) { struct rknpu_gem_object *rknpu_obj = to_rknpu_obj(obj); int ret = -EINVAL; @@ -1143,8 +1440,12 @@ rknpu_gem_prime_import_sg_table(struct drm_device *dev, goto err; } +#if KERNEL_VERSION(6, 1, 0) > LINUX_VERSION_CODE ret = drm_prime_sg_to_page_addr_arrays(sgt, rknpu_obj->pages, NULL, npages); +#else + ret = drm_prime_sg_to_page_array(sgt, rknpu_obj->pages, npages); +#endif if (ret < 0) goto err_free_large; @@ -1172,6 +1473,7 @@ err: return ERR_PTR(ret); } +#if KERNEL_VERSION(6, 1, 0) > LINUX_VERSION_CODE void *rknpu_gem_prime_vmap(struct drm_gem_object *obj) { struct rknpu_gem_object *rknpu_obj = to_rknpu_obj(obj); @@ -1187,6 +1489,35 @@ void rknpu_gem_prime_vunmap(struct drm_gem_object *obj, void *vaddr) { vunmap(vaddr); } +#else +int rknpu_gem_prime_vmap(struct drm_gem_object *obj, struct iosys_map *map) +{ + struct rknpu_gem_object *rknpu_obj = to_rknpu_obj(obj); + void *vaddr = NULL; + + if (!rknpu_obj->pages) + return -EINVAL; + + vaddr = vmap(rknpu_obj->pages, rknpu_obj->num_pages, VM_MAP, + PAGE_KERNEL); + if (!vaddr) + return -ENOMEM; + + iosys_map_set_vaddr(map, vaddr); + + return 0; +} + +void rknpu_gem_prime_vunmap(struct drm_gem_object *obj, struct iosys_map *map) +{ + struct rknpu_gem_object *rknpu_obj = to_rknpu_obj(obj); + + if (rknpu_obj->pages) { + vunmap(map->vaddr); + map->vaddr = NULL; + } +} +#endif int rknpu_gem_prime_mmap(struct drm_gem_object *obj, struct vm_area_struct *vma) { @@ -1199,14 +1530,150 @@ int rknpu_gem_prime_mmap(struct drm_gem_object *obj, struct vm_area_struct *vma) return rknpu_gem_mmap_obj(obj, vma); } +static int rknpu_cache_sync_with_sg(struct rknpu_device *rknpu_dev, + struct rknpu_gem_object *rknpu_obj, + unsigned long *length, + unsigned long *offset, uint32_t dir) +{ + struct scatterlist *s = NULL; + int i = 0; + int index = 0; + void __iomem *cache_start = 0; + unsigned long cache_length = 0; + + switch (rknpu_obj->core_mask) { + case RKNPU_CORE0_MASK: + index = 0; + break; + case RKNPU_CORE1_MASK: + index = 1; + break; + default: + break; + } + + for_each_sgtable_sg(rknpu_dev->cache_sgt[index], s, i) { + cache_start = rknpu_dev->nbuf_base_io + s->offset; + cache_length = (*offset + *length) <= s->length ? + *length : + s->length - *offset; + if (dir & RKNPU_MEM_SYNC_TO_DEVICE) { +#if KERNEL_VERSION(6, 1, 0) > LINUX_VERSION_CODE + __dma_map_area(cache_start, cache_length, + DMA_TO_DEVICE); +#else + dcache_clean_poc((unsigned long)cache_start, + (unsigned long)cache_start + + cache_length); +#endif + } + + if (dir & RKNPU_MEM_SYNC_FROM_DEVICE) { +#if KERNEL_VERSION(6, 1, 0) > LINUX_VERSION_CODE + __dma_unmap_area(cache_start, cache_length, + DMA_FROM_DEVICE); +#else + dcache_inval_poc((unsigned long)cache_start, + (unsigned long)cache_start + + cache_length); +#endif + } + + *length = (*offset + *length) <= s->length ? + 0 : + *length - cache_length; + *offset = 0; + + if (*length == 0) + break; + } + + return 0; +} + +static int rknpu_cache_sync(struct rknpu_gem_object *rknpu_obj, + unsigned long *length, unsigned long *offset, + enum rknpu_cache_type cache_type, uint32_t dir) +{ + struct drm_gem_object *obj = &rknpu_obj->base; + struct rknpu_device *rknpu_dev = obj->dev->dev_private; + void __iomem *cache_base_io = NULL; + unsigned long cache_offset = 0; + unsigned long cache_size = 0; + void __iomem *cache_start = 0; + unsigned long cache_length = 0; + + switch (cache_type) { + case RKNPU_CACHE_SRAM: + cache_base_io = rknpu_dev->sram_base_io; + cache_offset = rknpu_obj->sram_obj->range_start * + rknpu_dev->sram_mm->chunk_size; + cache_size = rknpu_obj->sram_size; + break; + case RKNPU_CACHE_NBUF: + cache_base_io = rknpu_dev->nbuf_base_io; + cache_offset = 0; + cache_size = rknpu_obj->nbuf_size; + break; + default: + LOG_ERROR("Unknown rknpu_cache_type: %d", cache_type); + return -EINVAL; + } + + if (*offset >= cache_size) { + *offset -= cache_size; + return 0; + } + + if (!rknpu_obj->cache_with_sgt) { + cache_start = cache_base_io + cache_offset; + cache_length = (*offset + *length) <= cache_size ? + *length : + cache_size - *offset; + if (dir & RKNPU_MEM_SYNC_TO_DEVICE) { +#if KERNEL_VERSION(6, 1, 0) > LINUX_VERSION_CODE + __dma_map_area(cache_start, cache_length, + DMA_TO_DEVICE); +#else + dcache_clean_poc((unsigned long)cache_start, + (unsigned long)cache_start + + cache_length); +#endif + } + + if (dir & RKNPU_MEM_SYNC_FROM_DEVICE) { +#if KERNEL_VERSION(6, 1, 0) > LINUX_VERSION_CODE + __dma_unmap_area(cache_start, cache_length, + DMA_FROM_DEVICE); +#else + dcache_inval_poc((unsigned long)cache_start, + (unsigned long)cache_start + + cache_length); +#endif + } + + *length = (*offset + *length) <= cache_size ? + 0 : + *length - cache_length; + *offset = 0; + } else { + rknpu_cache_sync_with_sg(rknpu_dev, rknpu_obj, length, offset, + dir); + } + + return 0; +} + int rknpu_gem_sync_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv) { struct rknpu_gem_object *rknpu_obj = NULL; + struct rknpu_device *rknpu_dev = dev->dev_private; struct rknpu_mem_sync *args = data; struct scatterlist *sg; + dma_addr_t sg_phys_addr; unsigned long length, offset = 0; - unsigned long sg_left, size = 0; + unsigned long sg_offset, sg_left, size = 0; unsigned long len = 0; int i; @@ -1217,6 +1684,12 @@ int rknpu_gem_sync_ioctl(struct drm_device *dev, void *data, if (!(rknpu_obj->flags & RKNPU_MEM_CACHEABLE)) return -EINVAL; + if (rknpu_iommu_domain_get_and_switch(rknpu_dev, + rknpu_obj->iommu_domain_id)) { + LOG_DEV_ERROR(rknpu_dev->dev, "%s error\n", __func__); + return -EINVAL; + } + if (!(rknpu_obj->flags & RKNPU_MEM_NON_CONTIGUOUS)) { if (args->flags & RKNPU_MEM_SYNC_TO_DEVICE) { dma_sync_single_range_for_device( @@ -1230,43 +1703,23 @@ int rknpu_gem_sync_ioctl(struct drm_device *dev, void *data, DMA_FROM_DEVICE); } } else { + WARN_ON(!rknpu_dev->fake_dev); + length = args->size; offset = args->offset; - if (IS_ENABLED(CONFIG_ROCKCHIP_RKNPU_SRAM) && + if (IS_ENABLED(CONFIG_NO_GKI) && + IS_ENABLED(CONFIG_ROCKCHIP_RKNPU_SRAM) && rknpu_obj->sram_size > 0) { - struct drm_gem_object *obj = &rknpu_obj->base; - struct rknpu_device *rknpu_dev = obj->dev->dev_private; - unsigned long sram_offset = - rknpu_obj->sram_obj->range_start * - rknpu_dev->sram_mm->chunk_size; - if ((offset + length) <= rknpu_obj->sram_size) { - __dma_map_area(rknpu_dev->sram_base_io + - offset + sram_offset, - length, DMA_TO_DEVICE); - __dma_unmap_area(rknpu_dev->sram_base_io + - offset + sram_offset, - length, DMA_FROM_DEVICE); - length = 0; - offset = 0; - } else if (offset >= rknpu_obj->sram_size) { - offset -= rknpu_obj->sram_size; - } else { - unsigned long sram_length = - rknpu_obj->sram_size - offset; - __dma_map_area(rknpu_dev->sram_base_io + - offset + sram_offset, - sram_length, DMA_TO_DEVICE); - __dma_unmap_area(rknpu_dev->sram_base_io + - offset + sram_offset, - sram_length, DMA_FROM_DEVICE); - length -= sram_length; - offset = 0; - } + rknpu_cache_sync(rknpu_obj, &length, &offset, + RKNPU_CACHE_SRAM, args->flags); + } else if (IS_ENABLED(CONFIG_NO_GKI) && + rknpu_obj->nbuf_size > 0) { + rknpu_cache_sync(rknpu_obj, &length, &offset, + RKNPU_CACHE_NBUF, args->flags); } - for_each_sg(rknpu_obj->sgt->sgl, sg, rknpu_obj->sgt->nents, - i) { + for_each_sg(rknpu_obj->sgt->sgl, sg, rknpu_obj->sgt->nents, i) { if (length == 0) break; @@ -1274,17 +1727,23 @@ int rknpu_gem_sync_ioctl(struct drm_device *dev, void *data, if (len <= offset) continue; + sg_phys_addr = sg_phys(sg); + sg_left = len - offset; + sg_offset = sg->length - sg_left; + size = (length < sg_left) ? length : sg_left; if (args->flags & RKNPU_MEM_SYNC_TO_DEVICE) { - dma_sync_sg_for_device(dev->dev, sg, 1, - DMA_TO_DEVICE); + dma_sync_single_range_for_device( + rknpu_dev->fake_dev, sg_phys_addr, + sg_offset, size, DMA_TO_DEVICE); } if (args->flags & RKNPU_MEM_SYNC_FROM_DEVICE) { - dma_sync_sg_for_cpu(dev->dev, sg, 1, - DMA_FROM_DEVICE); + dma_sync_single_range_for_cpu( + rknpu_dev->fake_dev, sg_phys_addr, + sg_offset, size, DMA_FROM_DEVICE); } offset += size; @@ -1292,5 +1751,7 @@ int rknpu_gem_sync_ioctl(struct drm_device *dev, void *data, } } + rknpu_iommu_domain_put(rknpu_dev); + return 0; } diff --git a/target/linux/rockchip/rk35xx/files/drivers/rknpu/rknpu_iommu.c b/target/linux/rockchip/rk35xx/files/drivers/rknpu/rknpu_iommu.c new file mode 100644 index 0000000000..efa97f39c8 --- /dev/null +++ b/target/linux/rockchip/rk35xx/files/drivers/rknpu/rknpu_iommu.c @@ -0,0 +1,619 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) Rockchip Electronics Co.Ltd + * Author: Felix Zeng + */ + +#include +#include +#include + +#include "rknpu_iommu.h" + +#define RKNPU_SWITCH_DOMAIN_WAIT_TIME_MS 6000 + +dma_addr_t rknpu_iommu_dma_alloc_iova(struct iommu_domain *domain, size_t size, + u64 dma_limit, struct device *dev, + bool size_aligned) +{ + struct rknpu_iommu_dma_cookie *cookie = (void *)domain->iova_cookie; + struct iova_domain *iovad = &cookie->iovad; + unsigned long shift, iova_len, iova = 0; + unsigned long limit_pfn; + struct iova *new_iova = NULL; + bool alloc_fast = size_aligned; + + shift = iova_shift(iovad); + iova_len = size >> shift; + +#if KERNEL_VERSION(6, 1, 0) > LINUX_VERSION_CODE + /* + * Freeing non-power-of-two-sized allocations back into the IOVA caches + * will come back to bite us badly, so we have to waste a bit of space + * rounding up anything cacheable to make sure that can't happen. The + * order of the unadjusted size will still match upon freeing. + */ + if (iova_len < (1 << (IOVA_RANGE_CACHE_MAX_SIZE - 1))) + iova_len = roundup_pow_of_two(iova_len); +#endif + +#if (KERNEL_VERSION(5, 10, 0) <= LINUX_VERSION_CODE) + dma_limit = min_not_zero(dma_limit, dev->bus_dma_limit); +#else + if (dev->bus_dma_mask) + dma_limit &= dev->bus_dma_mask; +#endif + + if (domain->geometry.force_aperture) + dma_limit = + min_t(u64, dma_limit, domain->geometry.aperture_end); + +#if (KERNEL_VERSION(5, 4, 0) <= LINUX_VERSION_CODE) + limit_pfn = dma_limit >> shift; +#else + limit_pfn = min_t(dma_addr_t, dma_limit >> shift, iovad->end_pfn); +#endif + + if (alloc_fast) { + iova = alloc_iova_fast(iovad, iova_len, limit_pfn, true); + } else { + new_iova = alloc_iova(iovad, iova_len, limit_pfn, size_aligned); + if (!new_iova) + return 0; + iova = new_iova->pfn_lo; + } + + return (dma_addr_t)iova << shift; +} + +void rknpu_iommu_dma_free_iova(struct rknpu_iommu_dma_cookie *cookie, + dma_addr_t iova, size_t size, bool size_aligned) +{ + struct iova_domain *iovad = &cookie->iovad; + bool alloc_fast = size_aligned; + + if (alloc_fast) + free_iova_fast(iovad, iova_pfn(iovad, iova), + size >> iova_shift(iovad)); + else + free_iova(iovad, iova_pfn(iovad, iova)); +} + +static int rknpu_dma_info_to_prot(enum dma_data_direction dir, bool coherent) +{ + int prot = coherent ? IOMMU_CACHE : 0; + + switch (dir) { + case DMA_BIDIRECTIONAL: + return prot | IOMMU_READ | IOMMU_WRITE; + case DMA_TO_DEVICE: + return prot | IOMMU_READ; + case DMA_FROM_DEVICE: + return prot | IOMMU_WRITE; + default: + return 0; + } +} + +/* + * Prepare a successfully-mapped scatterlist to give back to the caller. + * + * At this point the segments are already laid out by iommu_dma_map_sg() to + * avoid individually crossing any boundaries, so we merely need to check a + * segment's start address to avoid concatenating across one. + */ +static int __finalise_sg(struct device *dev, struct scatterlist *sg, int nents, + dma_addr_t dma_addr) +{ + struct scatterlist *s, *cur = sg; + unsigned long seg_mask = dma_get_seg_boundary(dev); + unsigned int cur_len = 0, max_len = dma_get_max_seg_size(dev); + int i, count = 0; + + for_each_sg(sg, s, nents, i) { + /* Restore this segment's original unaligned fields first */ +#if KERNEL_VERSION(6, 1, 0) <= LINUX_VERSION_CODE + dma_addr_t s_dma_addr = sg_dma_address(s); +#endif + unsigned int s_iova_off = sg_dma_address(s); + unsigned int s_length = sg_dma_len(s); + unsigned int s_iova_len = s->length; + + sg_dma_address(s) = DMA_MAPPING_ERROR; + sg_dma_len(s) = 0; + +#if KERNEL_VERSION(6, 1, 0) <= LINUX_VERSION_CODE + if (sg_is_dma_bus_address(s)) { + if (i > 0) + cur = sg_next(cur); + + sg_dma_unmark_bus_address(s); + sg_dma_address(cur) = s_dma_addr; + sg_dma_len(cur) = s_length; + sg_dma_mark_bus_address(cur); + count++; + cur_len = 0; + continue; + } +#endif + + s->offset += s_iova_off; + s->length = s_length; + + /* + * Now fill in the real DMA data. If... + * - there is a valid output segment to append to + * - and this segment starts on an IOVA page boundary + * - but doesn't fall at a segment boundary + * - and wouldn't make the resulting output segment too long + */ + if (cur_len && !s_iova_off && (dma_addr & seg_mask) && + (max_len - cur_len >= s_length)) { + /* ...then concatenate it with the previous one */ + cur_len += s_length; + } else { + /* Otherwise start the next output segment */ + if (i > 0) + cur = sg_next(cur); + cur_len = s_length; + count++; + + sg_dma_address(cur) = dma_addr + s_iova_off; + } + + sg_dma_len(cur) = cur_len; + dma_addr += s_iova_len; + + if (s_length + s_iova_off < s_iova_len) + cur_len = 0; + } + return count; +} + +/* + * If mapping failed, then just restore the original list, + * but making sure the DMA fields are invalidated. + */ +static void __invalidate_sg(struct scatterlist *sg, int nents) +{ + struct scatterlist *s; + int i; + +#if KERNEL_VERSION(6, 1, 0) <= LINUX_VERSION_CODE + for_each_sg(sg, s, nents, i) { + if (sg_is_dma_bus_address(s)) { + sg_dma_unmark_bus_address(s); + } else { + if (sg_dma_address(s) != DMA_MAPPING_ERROR) + s->offset += sg_dma_address(s); + if (sg_dma_len(s)) + s->length = sg_dma_len(s); + } + sg_dma_address(s) = DMA_MAPPING_ERROR; + sg_dma_len(s) = 0; + } +#else + for_each_sg(sg, s, nents, i) { + if (sg_dma_address(s) != DMA_MAPPING_ERROR) + s->offset += sg_dma_address(s); + if (sg_dma_len(s)) + s->length = sg_dma_len(s); + sg_dma_address(s) = DMA_MAPPING_ERROR; + sg_dma_len(s) = 0; + } +#endif +} + +int rknpu_iommu_dma_map_sg(struct device *dev, struct scatterlist *sg, + int nents, enum dma_data_direction dir, + bool iova_aligned) +{ + struct iommu_domain *domain = iommu_get_domain_for_dev(dev); + struct rknpu_iommu_dma_cookie *cookie = (void *)domain->iova_cookie; + struct iova_domain *iovad = &cookie->iovad; + struct scatterlist *s = NULL, *prev = NULL; + int prot = rknpu_dma_info_to_prot(dir, dev_is_dma_coherent(dev)); + dma_addr_t iova; + unsigned long iova_len = 0; + unsigned long mask = dma_get_seg_boundary(dev); + ssize_t ret = -EINVAL; + int i = 0; + + if (iova_aligned) + return dma_map_sg(dev, sg, nents, dir); + + /* + * Work out how much IOVA space we need, and align the segments to + * IOVA granules for the IOMMU driver to handle. With some clever + * trickery we can modify the list in-place, but reversibly, by + * stashing the unaligned parts in the as-yet-unused DMA fields. + */ + for_each_sg(sg, s, nents, i) { + size_t s_iova_off = iova_offset(iovad, s->offset); + size_t s_length = s->length; + size_t pad_len = (mask - iova_len + 1) & mask; + + sg_dma_address(s) = s_iova_off; + sg_dma_len(s) = s_length; + s->offset -= s_iova_off; + s_length = iova_align(iovad, s_length + s_iova_off); + s->length = s_length; + + /* + * Due to the alignment of our single IOVA allocation, we can + * depend on these assumptions about the segment boundary mask: + * - If mask size >= IOVA size, then the IOVA range cannot + * possibly fall across a boundary, so we don't care. + * - If mask size < IOVA size, then the IOVA range must start + * exactly on a boundary, therefore we can lay things out + * based purely on segment lengths without needing to know + * the actual addresses beforehand. + * - The mask must be a power of 2, so pad_len == 0 if + * iova_len == 0, thus we cannot dereference prev the first + * time through here (i.e. before it has a meaningful value). + */ + if (pad_len && pad_len < s_length - 1) { + prev->length += pad_len; + iova_len += pad_len; + } + + iova_len += s_length; + prev = s; + } + + if (!iova_len) { + ret = __finalise_sg(dev, sg, nents, 0); + goto out; + } + + iova = rknpu_iommu_dma_alloc_iova(domain, iova_len, dma_get_mask(dev), + dev, iova_aligned); + if (!iova) { + ret = -ENOMEM; + LOG_ERROR("failed to allocate IOVA: %zd\n", ret); + goto out_restore_sg; + } + + ret = iommu_map_sg(domain, iova, sg, nents, prot); + if (ret < 0 || ret < iova_len) { + LOG_ERROR("failed to map SG: %zd\n", ret); + goto out_free_iova; + } + + return __finalise_sg(dev, sg, nents, iova); + +out_free_iova: + rknpu_iommu_dma_free_iova(cookie, iova, iova_len, iova_aligned); +out_restore_sg: + __invalidate_sg(sg, nents); +out: + + if (ret < 0) + ret = 0; + + return ret; +} + +void rknpu_iommu_dma_unmap_sg(struct device *dev, struct scatterlist *sg, + int nents, enum dma_data_direction dir, + bool iova_aligned) +{ + struct iommu_domain *domain = iommu_get_domain_for_dev(dev); + struct rknpu_iommu_dma_cookie *cookie = (void *)domain->iova_cookie; + struct iova_domain *iovad = &cookie->iovad; + size_t iova_off = 0; + dma_addr_t end = 0, start = 0; + struct scatterlist *tmp = NULL; + dma_addr_t dma_addr = 0; + size_t size = 0; + int i = 0; + + if (iova_aligned) + return dma_unmap_sg(dev, sg, nents, dir); + +#if KERNEL_VERSION(6, 1, 0) <= LINUX_VERSION_CODE + /* + * The scatterlist segments are mapped into a single + * contiguous IOVA allocation, the start and end points + * just have to be determined. + */ + for_each_sg(sg, tmp, nents, i) { + if (sg_is_dma_bus_address(tmp)) { + sg_dma_unmark_bus_address(tmp); + continue; + } + + if (sg_dma_len(tmp) == 0) + break; + + start = sg_dma_address(tmp); + break; + } + + nents -= i; + for_each_sg(tmp, tmp, nents, i) { + if (sg_is_dma_bus_address(tmp)) { + sg_dma_unmark_bus_address(tmp); + continue; + } + + if (sg_dma_len(tmp) == 0) + break; + + end = sg_dma_address(tmp) + sg_dma_len(tmp); + } +#else + start = sg_dma_address(sg); + for_each_sg(sg_next(sg), tmp, nents - 1, i) { + if (sg_dma_len(tmp) == 0) + break; + sg = tmp; + } + end = sg_dma_address(sg) + sg_dma_len(sg); +#endif + + dma_addr = start; + size = end - start; + iova_off = iova_offset(iovad, start); + + if (end) { + dma_addr -= iova_off; + size = iova_align(iovad, size + iova_off); + iommu_unmap(domain, dma_addr, size); + rknpu_iommu_dma_free_iova(cookie, dma_addr, size, iova_aligned); + } +} + +#if defined(CONFIG_IOMMU_API) && defined(CONFIG_NO_GKI) + +#if KERNEL_VERSION(6, 1, 0) <= LINUX_VERSION_CODE +struct iommu_group { + struct kobject kobj; + struct kobject *devices_kobj; + struct list_head devices; +#ifdef __ANDROID_COMMON_KERNEL__ + struct xarray pasid_array; +#endif + struct mutex mutex; + void *iommu_data; + void (*iommu_data_release)(void *iommu_data); + char *name; + int id; + struct iommu_domain *default_domain; + struct iommu_domain *blocking_domain; + struct iommu_domain *domain; + struct list_head entry; + unsigned int owner_cnt; + void *owner; +}; +#else +struct iommu_group { + struct kobject kobj; + struct kobject *devices_kobj; + struct list_head devices; + struct mutex mutex; + struct blocking_notifier_head notifier; + void *iommu_data; + void (*iommu_data_release)(void *iommu_data); + char *name; + int id; + struct iommu_domain *default_domain; + struct iommu_domain *domain; + struct list_head entry; +}; +#endif + +int rknpu_iommu_init_domain(struct rknpu_device *rknpu_dev) +{ + // init domain 0 + if (!rknpu_dev->iommu_domains[0]) { + rknpu_dev->iommu_domain_id = 0; + rknpu_dev->iommu_domains[rknpu_dev->iommu_domain_id] = + iommu_get_domain_for_dev(rknpu_dev->dev); + rknpu_dev->iommu_domain_num = 1; + } + return 0; +} + +int rknpu_iommu_switch_domain(struct rknpu_device *rknpu_dev, int domain_id) +{ + struct iommu_domain *src_domain = NULL; + struct iommu_domain *dst_domain = NULL; + struct bus_type *bus = NULL; + int src_domain_id = 0; + int ret = -EINVAL; + + if (!rknpu_dev->iommu_en) + return -EINVAL; + + if (domain_id < 0 || domain_id > (RKNPU_MAX_IOMMU_DOMAIN_NUM - 1)) { + LOG_DEV_ERROR( + rknpu_dev->dev, + "invalid iommu domain id: %d, reuse domain id: %d\n", + domain_id, rknpu_dev->iommu_domain_id); + return -EINVAL; + } + + bus = rknpu_dev->dev->bus; + if (!bus) + return -EFAULT; + + src_domain_id = rknpu_dev->iommu_domain_id; + if (domain_id == src_domain_id) { + return 0; + } + + src_domain = iommu_get_domain_for_dev(rknpu_dev->dev); + if (src_domain != rknpu_dev->iommu_domains[src_domain_id]) { + LOG_DEV_ERROR( + rknpu_dev->dev, + "mismatch domain get from iommu_get_domain_for_dev\n"); + return -EINVAL; + } + + dst_domain = rknpu_dev->iommu_domains[domain_id]; + if (dst_domain != NULL) { + iommu_detach_device(src_domain, rknpu_dev->dev); + ret = iommu_attach_device(dst_domain, rknpu_dev->dev); + if (ret) { + LOG_DEV_ERROR( + rknpu_dev->dev, + "failed to attach dst iommu domain, id: %d, ret: %d\n", + domain_id, ret); + if (iommu_attach_device(src_domain, rknpu_dev->dev)) { + LOG_DEV_ERROR( + rknpu_dev->dev, + "failed to reattach src iommu domain, id: %d\n", + src_domain_id); + } + return ret; + } + rknpu_dev->iommu_domain_id = domain_id; + } else { + uint64_t dma_limit = 1ULL << 32; + + dst_domain = iommu_domain_alloc(bus); + if (!dst_domain) { + LOG_DEV_ERROR(rknpu_dev->dev, + "failed to allocate iommu domain\n"); + return -EIO; + } + // init domain iova_cookie + iommu_get_dma_cookie(dst_domain); + + iommu_detach_device(src_domain, rknpu_dev->dev); + ret = iommu_attach_device(dst_domain, rknpu_dev->dev); + if (ret) { + LOG_DEV_ERROR( + rknpu_dev->dev, + "failed to attach iommu domain, id: %d, ret: %d\n", + domain_id, ret); + iommu_domain_free(dst_domain); + return ret; + } + + // set domain type to dma domain + dst_domain->type |= __IOMMU_DOMAIN_DMA_API; + // iommu dma init domain + iommu_setup_dma_ops(rknpu_dev->dev, 0, dma_limit); + + rknpu_dev->iommu_domain_id = domain_id; + rknpu_dev->iommu_domains[domain_id] = dst_domain; + rknpu_dev->iommu_domain_num++; + } + + // reset default iommu domain + rknpu_dev->iommu_group->default_domain = dst_domain; + + LOG_INFO("switch iommu domain from %d to %d\n", src_domain_id, + domain_id); + + return ret; +} + +int rknpu_iommu_domain_get_and_switch(struct rknpu_device *rknpu_dev, + int domain_id) +{ + unsigned long timeout_jiffies = + msecs_to_jiffies(RKNPU_SWITCH_DOMAIN_WAIT_TIME_MS); + unsigned long start = jiffies; + int ret = -EINVAL; + + while (true) { + mutex_lock(&rknpu_dev->domain_lock); + + if (domain_id == rknpu_dev->iommu_domain_id) { + atomic_inc(&rknpu_dev->iommu_domain_refcount); + mutex_unlock(&rknpu_dev->domain_lock); + break; + } + + if (atomic_read(&rknpu_dev->iommu_domain_refcount) == 0) { + ret = rknpu_iommu_switch_domain(rknpu_dev, domain_id); + if (ret) { + LOG_DEV_ERROR( + rknpu_dev->dev, + "failed to switch iommu domain, id: %d, ret: %d\n", + domain_id, ret); + mutex_unlock(&rknpu_dev->domain_lock); + return ret; + } + atomic_inc(&rknpu_dev->iommu_domain_refcount); + mutex_unlock(&rknpu_dev->domain_lock); + break; + } + + mutex_unlock(&rknpu_dev->domain_lock); + + usleep_range(10, 100); + if (time_after(jiffies, start + timeout_jiffies)) { + LOG_DEV_ERROR( + rknpu_dev->dev, + "switch iommu domain time out, failed to switch iommu domain, id: %d\n", + domain_id); + return -EINVAL; + } + } + + return 0; +} + +int rknpu_iommu_domain_put(struct rknpu_device *rknpu_dev) +{ + atomic_dec(&rknpu_dev->iommu_domain_refcount); + + return 0; +} + +void rknpu_iommu_free_domains(struct rknpu_device *rknpu_dev) +{ + int i = 0; + + if (rknpu_iommu_domain_get_and_switch(rknpu_dev, 0)) { + LOG_DEV_ERROR(rknpu_dev->dev, "%s error\n", __func__); + return; + } + + for (i = 1; i < RKNPU_MAX_IOMMU_DOMAIN_NUM; i++) { + struct iommu_domain *domain = rknpu_dev->iommu_domains[i]; + + if (domain == NULL) + continue; + + iommu_detach_device(domain, rknpu_dev->dev); + iommu_domain_free(domain); + + rknpu_dev->iommu_domains[i] = NULL; + } + + rknpu_iommu_domain_put(rknpu_dev); +} + +#else + +int rknpu_iommu_init_domain(struct rknpu_device *rknpu_dev) +{ + return 0; +} + +int rknpu_iommu_switch_domain(struct rknpu_device *rknpu_dev, int domain_id) +{ + return 0; +} + +int rknpu_iommu_domain_get_and_switch(struct rknpu_device *rknpu_dev, + int domain_id) +{ + return 0; +} + +int rknpu_iommu_domain_put(struct rknpu_device *rknpu_dev) +{ + return 0; +} + +void rknpu_iommu_free_domains(struct rknpu_device *rknpu_dev) +{ +} + +#endif diff --git a/target/linux/rockchip/rk35xx/files/drivers/rknpu/rknpu_job.c b/target/linux/rockchip/rk35xx/files/drivers/rknpu/rknpu_job.c index 6be7824a90..23ed8e5cf8 100644 --- a/target/linux/rockchip/rk35xx/files/drivers/rknpu/rknpu_job.c +++ b/target/linux/rockchip/rk35xx/files/drivers/rknpu/rknpu_job.c @@ -14,8 +14,9 @@ #include "rknpu_reset.h" #include "rknpu_gem.h" #include "rknpu_fence.h" -#include "rknpu_job.h" #include "rknpu_mem.h" +#include "rknpu_iommu.h" +#include "rknpu_job.h" #define _REG_READ(base, offset) readl(base + (offset)) #define _REG_WRITE(base, value, offset) writel(value, base + (offset)) @@ -23,16 +24,25 @@ #define REG_READ(offset) _REG_READ(rknpu_core_base, offset) #define REG_WRITE(value, offset) _REG_WRITE(rknpu_core_base, value, offset) -static int rknpu_core_index(int core_mask) +static int rknpu_wait_core_index(int core_mask) { int index = 0; - if (core_mask & RKNPU_CORE0_MASK) + switch (core_mask) { + case RKNPU_CORE0_MASK: + case RKNPU_CORE0_MASK | RKNPU_CORE1_MASK: + case RKNPU_CORE0_MASK | RKNPU_CORE1_MASK | RKNPU_CORE2_MASK: index = 0; - else if (core_mask & RKNPU_CORE1_MASK) + break; + case RKNPU_CORE1_MASK: index = 1; - else if (core_mask & RKNPU_CORE2_MASK) + break; + case RKNPU_CORE2_MASK: index = 2; + break; + default: + break; + } return index; } @@ -58,14 +68,24 @@ static int rknpu_core_mask(int core_index) return core_mask; } -static int rknn_get_task_number(struct rknpu_job *job, int core_index) +static int rknpu_get_task_number(struct rknpu_job *job, int core_index) { + struct rknpu_device *rknpu_dev = job->rknpu_dev; int task_num = job->args->task_number; - if (job->use_core_num == 2) - task_num = job->args->subcore_task[core_index].task_number; - else if (job->use_core_num == 3) - task_num = job->args->subcore_task[core_index + 2].task_number; + if (core_index >= RKNPU_MAX_CORES || core_index < 0) { + LOG_ERROR("invalid rknpu core index: %d", core_index); + return 0; + } + + if (rknpu_dev->config->num_irqs > 1) { + if (job->use_core_num == 1 || job->use_core_num == 2) + task_num = + job->args->subcore_task[core_index].task_number; + else if (job->use_core_num == 3) + task_num = job->args->subcore_task[core_index + 2] + .task_number; + } return task_num; } @@ -109,11 +129,10 @@ static inline struct rknpu_job *rknpu_job_alloc(struct rknpu_device *rknpu_dev, struct rknpu_submit *args) { struct rknpu_job *job = NULL; + int i = 0; #ifdef CONFIG_ROCKCHIP_RKNPU_DRM_GEM struct rknpu_gem_object *task_obj = NULL; #endif - if (rknpu_dev->config->num_irqs == 1) - args->core_mask = RKNPU_CORE0_MASK; job = kzalloc(sizeof(*job), GFP_KERNEL); if (!job) @@ -124,8 +143,11 @@ static inline struct rknpu_job *rknpu_job_alloc(struct rknpu_device *rknpu_dev, job->use_core_num = (args->core_mask & RKNPU_CORE0_MASK) + ((args->core_mask & RKNPU_CORE1_MASK) >> 1) + ((args->core_mask & RKNPU_CORE2_MASK) >> 2); - job->run_count = job->use_core_num; - job->interrupt_count = job->use_core_num; + atomic_set(&job->run_count, job->use_core_num); + atomic_set(&job->interrupt_count, job->use_core_num); + job->iommu_domain_id = args->iommu_domain_id; + for (i = 0; i < rknpu_dev->config->num_irqs; i++) + atomic_set(&job->submit_count[i], 0); #ifdef CONFIG_ROCKCHIP_RKNPU_DRM_GEM task_obj = (struct rknpu_gem_object *)(uintptr_t)args->task_obj_addr; if (task_obj) @@ -157,10 +179,12 @@ static inline int rknpu_job_wait(struct rknpu_job *job) struct rknpu_submit *args = job->args; struct rknpu_task *last_task = NULL; struct rknpu_subcore_data *subcore_data = NULL; + struct rknpu_job *entry, *q; void __iomem *rknpu_core_base = NULL; - int core_index = rknpu_core_index(job->args->core_mask); + int core_index = rknpu_wait_core_index(job->args->core_mask); unsigned long flags; int wait_count = 0; + bool continue_wait = false; int ret = -EINVAL; int i = 0; @@ -171,32 +195,50 @@ static inline int rknpu_job_wait(struct rknpu_job *job) job->flags & RKNPU_JOB_DONE || rknpu_dev->soft_reseting, msecs_to_jiffies(args->timeout)); + if (++wait_count >= 3) break; - } while (ret == 0 && job->in_queue[core_index]); - if (job->in_queue[core_index]) { - spin_lock_irqsave(&rknpu_dev->lock, flags); - subcore_data->task_num -= rknn_get_task_number(job, core_index); - if (job->use_core_num == 1) { - list_del_init(&job->head[core_index]); - job->in_queue[core_index] = false; - } else if (job->use_core_num > 1) { - for (i = 0; i < job->use_core_num; i++) { - if (job->in_queue[i]) { - list_del_init(&job->head[i]); - job->in_queue[i] = false; + if (ret == 0) { + int64_t elapse_time_us = 0; + spin_lock_irqsave(&rknpu_dev->irq_lock, flags); + elapse_time_us = ktime_us_delta(ktime_get(), + job->hw_commit_time); + continue_wait = + job->hw_commit_time == 0 ? + true : + (elapse_time_us < args->timeout * 1000); + spin_unlock_irqrestore(&rknpu_dev->irq_lock, flags); + LOG_ERROR( + "job: %p, mask: %#x, job iommu domain id: %d, dev iommu domain id: %d, wait_count: %d, continue wait: %d, commit elapse time: %lldus, wait time: %lldus, timeout: %uus\n", + job, args->core_mask, job->iommu_domain_id, + rknpu_dev->iommu_domain_id, wait_count, + continue_wait, + (job->hw_commit_time == 0 ? 0 : elapse_time_us), + ktime_us_delta(ktime_get(), job->timestamp), + args->timeout * 1000); + } + } while (ret == 0 && continue_wait); + + last_task = job->last_task; + if (!last_task) { + spin_lock_irqsave(&rknpu_dev->irq_lock, flags); + for (i = 0; i < job->use_core_num; i++) { + subcore_data = &rknpu_dev->subcore_datas[i]; + list_for_each_entry_safe( + entry, q, &subcore_data->todo_list, head[i]) { + if (entry == job) { + list_del(&job->head[i]); + break; } } } - spin_unlock_irqrestore(&rknpu_dev->lock, flags); + spin_unlock_irqrestore(&rknpu_dev->irq_lock, flags); + + LOG_ERROR("job commit failed\n"); return ret < 0 ? ret : -EINVAL; } - last_task = job->last_task; - if (!last_task) - return ret < 0 ? ret : -EINVAL; - last_task->int_status = job->int_status[core_index]; if (ret <= 0) { @@ -213,7 +255,7 @@ static inline int rknpu_job_wait(struct rknpu_job *job) LOG_ERROR( "failed to wait job, task counter: %d, flags: %#x, ret = %d, elapsed time: %lldus\n", args->task_counter, args->flags, ret, - ktime_to_us(ktime_sub(ktime_get(), job->timestamp))); + ktime_us_delta(ktime_get(), job->timestamp)); return ret < 0 ? ret : -ETIMEDOUT; } @@ -222,11 +264,13 @@ static inline int rknpu_job_wait(struct rknpu_job *job) return -EINVAL; args->task_counter = args->task_number; + args->hw_elapse_time = job->hw_elapse_time; return 0; } -static inline int rknpu_job_commit_pc(struct rknpu_job *job, int core_index) +static inline int rknpu_job_subcore_commit_pc(struct rknpu_job *job, + int core_index) { struct rknpu_device *rknpu_dev = job->rknpu_dev; struct rknpu_submit *args = job->args; @@ -243,15 +287,20 @@ static inline int rknpu_job_commit_pc(struct rknpu_job *job, int core_index) struct rknpu_task *last_task = NULL; void __iomem *rknpu_core_base = rknpu_dev->base[core_index]; int task_start = args->task_start; - int task_end = args->task_start + args->task_number - 1; + int task_end; int task_number = args->task_number; int task_pp_en = args->flags & RKNPU_JOB_PINGPONG ? 1 : 0; int pc_data_amount_scale = rknpu_dev->config->pc_data_amount_scale; int pc_task_number_bits = rknpu_dev->config->pc_task_number_bits; int i = 0; + int submit_index = atomic_read(&job->submit_count[core_index]); + int max_submit_number = rknpu_dev->config->max_submit_number; + unsigned long flags; - if (!task_obj) - return -EINVAL; + if (!task_obj) { + job->ret = -EINVAL; + return job->ret; + } if (rknpu_dev->config->num_irqs > 1) { for (i = 0; i < rknpu_dev->config->num_irqs; i++) { @@ -261,38 +310,44 @@ static inline int rknpu_job_commit_pc(struct rknpu_job *job, int core_index) } } - if (job->use_core_num == 1) { + switch (job->use_core_num) { + case 1: + case 2: task_start = args->subcore_task[core_index].task_start; - task_end = args->subcore_task[core_index].task_start + - args->subcore_task[core_index].task_number - - 1; task_number = args->subcore_task[core_index].task_number; - } else if (job->use_core_num == 2) { - task_start = args->subcore_task[core_index].task_start; - task_end = args->subcore_task[core_index].task_start + - args->subcore_task[core_index].task_number - - 1; - task_number = - args->subcore_task[core_index].task_number; - } else if (job->use_core_num == 3) { + break; + case 3: task_start = args->subcore_task[core_index + 2].task_start; - task_end = - args->subcore_task[core_index + 2].task_start + - args->subcore_task[core_index + 2].task_number - - 1; task_number = args->subcore_task[core_index + 2].task_number; + break; + default: + LOG_ERROR("Unknown use core num %d\n", + job->use_core_num); + break; } } + task_start = task_start + submit_index * max_submit_number; + task_number = task_number - submit_index * max_submit_number; + task_number = task_number > max_submit_number ? max_submit_number : + task_number; + task_end = task_start + task_number - 1; + task_base = task_obj->kv_addr; first_task = &task_base[task_start]; last_task = &task_base[task_end]; - REG_WRITE(first_task->regcmd_addr, RKNPU_OFFSET_PC_DATA_ADDR); + if (rknpu_dev->config->pc_dma_ctrl) { + spin_lock_irqsave(&rknpu_dev->irq_lock, flags); + REG_WRITE(first_task->regcmd_addr, RKNPU_OFFSET_PC_DATA_ADDR); + spin_unlock_irqrestore(&rknpu_dev->irq_lock, flags); + } else { + REG_WRITE(first_task->regcmd_addr, RKNPU_OFFSET_PC_DATA_ADDR); + } REG_WRITE((first_task->regcfg_amount + RKNPU_PC_DATA_EXTRA_AMOUNT + pc_data_amount_scale - 1) / @@ -319,19 +374,56 @@ static inline int rknpu_job_commit_pc(struct rknpu_job *job, int core_index) return 0; } -static int rknpu_job_commit(struct rknpu_job *job, int core_index) +static inline int rknpu_job_subcore_commit(struct rknpu_job *job, + int core_index) { struct rknpu_device *rknpu_dev = job->rknpu_dev; struct rknpu_submit *args = job->args; void __iomem *rknpu_core_base = rknpu_dev->base[core_index]; + unsigned long flags; // switch to slave mode - REG_WRITE(0x1, RKNPU_OFFSET_PC_DATA_ADDR); + if (rknpu_dev->config->pc_dma_ctrl) { + spin_lock_irqsave(&rknpu_dev->irq_lock, flags); + REG_WRITE(0x1, RKNPU_OFFSET_PC_DATA_ADDR); + spin_unlock_irqrestore(&rknpu_dev->irq_lock, flags); + } else { + REG_WRITE(0x1, RKNPU_OFFSET_PC_DATA_ADDR); + } - if (!(args->flags & RKNPU_JOB_PC)) - return -EINVAL; + if (!(args->flags & RKNPU_JOB_PC)) { + job->ret = -EINVAL; + return job->ret; + } - return rknpu_job_commit_pc(job, core_index); + return rknpu_job_subcore_commit_pc(job, core_index); +} + +static void rknpu_job_commit(struct rknpu_job *job) +{ + switch (job->args->core_mask) { + case RKNPU_CORE0_MASK: + rknpu_job_subcore_commit(job, 0); + break; + case RKNPU_CORE1_MASK: + rknpu_job_subcore_commit(job, 1); + break; + case RKNPU_CORE2_MASK: + rknpu_job_subcore_commit(job, 2); + break; + case RKNPU_CORE0_MASK | RKNPU_CORE1_MASK: + rknpu_job_subcore_commit(job, 0); + rknpu_job_subcore_commit(job, 1); + break; + case RKNPU_CORE0_MASK | RKNPU_CORE1_MASK | RKNPU_CORE2_MASK: + rknpu_job_subcore_commit(job, 0); + rknpu_job_subcore_commit(job, 1); + rknpu_job_subcore_commit(job, 2); + break; + default: + LOG_ERROR("Unknown core mask: %d\n", job->args->core_mask); + break; + } } static void rknpu_job_next(struct rknpu_device *rknpu_dev, int core_index) @@ -356,42 +448,45 @@ static void rknpu_job_next(struct rknpu_device *rknpu_dev, int core_index) head[core_index]); list_del_init(&job->head[core_index]); - job->in_queue[core_index] = false; subcore_data->job = job; - job->run_count--; - job->hw_recoder_time = ktime_get(); + job->hw_commit_time = ktime_get(); + job->hw_recoder_time = job->hw_commit_time; spin_unlock_irqrestore(&rknpu_dev->irq_lock, flags); - if (job->run_count == 0) { - if (job->args->core_mask & RKNPU_CORE0_MASK) - job->ret = rknpu_job_commit(job, 0); - if (job->args->core_mask & RKNPU_CORE1_MASK) - job->ret = rknpu_job_commit(job, 1); - if (job->args->core_mask & RKNPU_CORE2_MASK) - job->ret = rknpu_job_commit(job, 2); - } + if (atomic_dec_and_test(&job->run_count)) + rknpu_job_commit(job); } static void rknpu_job_done(struct rknpu_job *job, int ret, int core_index) { struct rknpu_device *rknpu_dev = job->rknpu_dev; struct rknpu_subcore_data *subcore_data = NULL; + ktime_t now; unsigned long flags; - ktime_t now = ktime_get(); + int max_submit_number = rknpu_dev->config->max_submit_number; + + if (atomic_inc_return(&job->submit_count[core_index]) < + (rknpu_get_task_number(job, core_index) + max_submit_number - 1) / + max_submit_number) { + rknpu_job_subcore_commit(job, core_index); + return; + } subcore_data = &rknpu_dev->subcore_datas[core_index]; spin_lock_irqsave(&rknpu_dev->irq_lock, flags); subcore_data->job = NULL; - subcore_data->task_num -= rknn_get_task_number(job, core_index); - job->interrupt_count--; - subcore_data->timer.busy_time += - ktime_us_delta(now, job->hw_recoder_time); + subcore_data->task_num -= rknpu_get_task_number(job, core_index); + now = ktime_get(); + job->hw_elapse_time = ktime_sub(now, job->hw_commit_time); + subcore_data->timer.busy_time += ktime_sub(now, job->hw_recoder_time); spin_unlock_irqrestore(&rknpu_dev->irq_lock, flags); - if (job->interrupt_count == 0) { + if (atomic_dec_and_test(&job->interrupt_count)) { int use_core_num = job->use_core_num; + rknpu_iommu_domain_put(rknpu_dev); + job->flags |= RKNPU_JOB_DONE; job->ret = ret; @@ -410,47 +505,41 @@ static void rknpu_job_done(struct rknpu_job *job, int ret, int core_index) rknpu_job_next(rknpu_dev, core_index); } +static int rknpu_schedule_core_index(struct rknpu_device *rknpu_dev) +{ + int core_num = rknpu_dev->config->num_irqs; + int task_num = rknpu_dev->subcore_datas[0].task_num; + int core_index = 0; + int i = 0; + + for (i = 1; i < core_num; i++) { + if (task_num > rknpu_dev->subcore_datas[i].task_num) { + core_index = i; + task_num = rknpu_dev->subcore_datas[i].task_num; + } + } + + return core_index; +} + static void rknpu_job_schedule(struct rknpu_job *job) { struct rknpu_device *rknpu_dev = job->rknpu_dev; struct rknpu_subcore_data *subcore_data = NULL; int i = 0, core_index = 0; unsigned long flags; - int task_num_list[3] = { 0, 1, 2 }; - int tmp = 0; - - if ((job->args->core_mask & 0x07) == RKNPU_CORE_AUTO_MASK) { - if (rknpu_dev->subcore_datas[0].task_num > - rknpu_dev->subcore_datas[1].task_num) { - tmp = task_num_list[1]; - task_num_list[1] = task_num_list[0]; - task_num_list[0] = tmp; - } - if (rknpu_dev->subcore_datas[task_num_list[0]].task_num > - rknpu_dev->subcore_datas[2].task_num) { - tmp = task_num_list[2]; - task_num_list[2] = task_num_list[1]; - task_num_list[1] = task_num_list[0]; - task_num_list[0] = tmp; - } else if (rknpu_dev->subcore_datas[task_num_list[1]].task_num > - rknpu_dev->subcore_datas[2].task_num) { - tmp = task_num_list[2]; - task_num_list[2] = task_num_list[1]; - task_num_list[1] = tmp; - } - if (!rknpu_dev->subcore_datas[task_num_list[0]].job) - core_index = task_num_list[0]; - else if (!rknpu_dev->subcore_datas[task_num_list[1]].job) - core_index = task_num_list[1]; - else if (!rknpu_dev->subcore_datas[task_num_list[2]].job) - core_index = task_num_list[2]; - else - core_index = task_num_list[0]; + if (job->args->core_mask == RKNPU_CORE_AUTO_MASK) { + core_index = rknpu_schedule_core_index(rknpu_dev); job->args->core_mask = rknpu_core_mask(core_index); job->use_core_num = 1; - job->interrupt_count = 1; - job->run_count = 1; + atomic_set(&job->run_count, job->use_core_num); + atomic_set(&job->interrupt_count, job->use_core_num); + } + + if (rknpu_iommu_domain_get_and_switch(rknpu_dev, job->iommu_domain_id)) { + job->ret = -EINVAL; + return; } spin_lock_irqsave(&rknpu_dev->irq_lock, flags); @@ -458,8 +547,7 @@ static void rknpu_job_schedule(struct rknpu_job *job) if (job->args->core_mask & rknpu_core_mask(i)) { subcore_data = &rknpu_dev->subcore_datas[i]; list_add_tail(&job->head[i], &subcore_data->todo_list); - subcore_data->task_num += rknn_get_task_number(job, i); - job->in_queue[i] = true; + subcore_data->task_num += rknpu_get_task_number(job, i); } } spin_unlock_irqrestore(&rknpu_dev->irq_lock, flags); @@ -474,41 +562,51 @@ static void rknpu_job_abort(struct rknpu_job *job) { struct rknpu_device *rknpu_dev = job->rknpu_dev; struct rknpu_subcore_data *subcore_data = NULL; - int core_index = rknpu_core_index(job->args->core_mask); - void __iomem *rknpu_core_base = rknpu_dev->base[core_index]; unsigned long flags; int i = 0; + rknpu_iommu_domain_put(rknpu_dev); + msleep(100); + spin_lock_irqsave(&rknpu_dev->irq_lock, flags); for (i = 0; i < rknpu_dev->config->num_irqs; i++) { if (job->args->core_mask & rknpu_core_mask(i)) { subcore_data = &rknpu_dev->subcore_datas[i]; - spin_lock_irqsave(&rknpu_dev->irq_lock, flags); if (job == subcore_data->job && !job->irq_entry[i]) { subcore_data->job = NULL; subcore_data->task_num -= - rknn_get_task_number(job, i); + rknpu_get_task_number(job, i); } - spin_unlock_irqrestore(&rknpu_dev->irq_lock, flags); } } + spin_unlock_irqrestore(&rknpu_dev->irq_lock, flags); if (job->ret == -ETIMEDOUT) { - LOG_ERROR( - "job timeout, flags: %#x, irq status: %#x, raw status: %#x, require mask: %#x, task counter: %#x, elapsed time: %lldus\n", - job->flags, REG_READ(RKNPU_OFFSET_INT_STATUS), - REG_READ(RKNPU_OFFSET_INT_RAW_STATUS), - job->int_mask[core_index], - (REG_READ(rknpu_dev->config->pc_task_status_offset) & - rknpu_dev->config->pc_task_number_mask), - ktime_to_us(ktime_sub(ktime_get(), job->timestamp))); + LOG_ERROR("job timeout, flags: %#x:\n", job->flags); + for (i = 0; i < rknpu_dev->config->num_irqs; i++) { + if (job->args->core_mask & rknpu_core_mask(i)) { + void __iomem *rknpu_core_base = + rknpu_dev->base[i]; + LOG_ERROR( + "\tcore %d irq status: %#x, raw status: %#x, require mask: %#x, task counter: %#x, elapsed time: %lldus\n", + i, REG_READ(RKNPU_OFFSET_INT_STATUS), + REG_READ(RKNPU_OFFSET_INT_RAW_STATUS), + job->int_mask[i], + (REG_READ( + rknpu_dev->config + ->pc_task_status_offset) & + rknpu_dev->config->pc_task_number_mask), + ktime_us_delta(ktime_get(), + job->timestamp)); + } + } rknpu_soft_reset(rknpu_dev); } else { LOG_ERROR( "job abort, flags: %#x, ret: %d, elapsed time: %lldus\n", job->flags, job->ret, - ktime_to_us(ktime_sub(ktime_get(), job->timestamp))); + ktime_us_delta(ktime_get(), job->timestamp)); } rknpu_job_cleanup(job); @@ -603,7 +701,6 @@ static void rknpu_job_timeout_clean(struct rknpu_device *rknpu_dev, { struct rknpu_job *job = NULL; unsigned long flags; - ktime_t now = ktime_get(); struct rknpu_subcore_data *subcore_data = NULL; int i = 0; @@ -612,7 +709,7 @@ static void rknpu_job_timeout_clean(struct rknpu_device *rknpu_dev, subcore_data = &rknpu_dev->subcore_datas[i]; job = subcore_data->job; if (job && - ktime_to_ms(ktime_sub(now, job->timestamp)) >= + ktime_us_delta(ktime_get(), job->timestamp) >= job->args->timeout) { rknpu_soft_reset(rknpu_dev); @@ -634,7 +731,6 @@ static void rknpu_job_timeout_clean(struct rknpu_device *rknpu_dev, struct rknpu_job, head[i]); list_del_init(&job->head[i]); - job->in_queue[i] = false; } else { job = NULL; } @@ -658,6 +754,11 @@ static int rknpu_submit(struct rknpu_device *rknpu_dev, return -EINVAL; } + if (args->core_mask > rknpu_dev->config->core_mask) { + LOG_ERROR("invalid rknpu core mask: %#x", args->core_mask); + return -EINVAL; + } + job = rknpu_job_alloc(rknpu_dev, args); if (!job) { LOG_ERROR("failed to allocate rknpu job!\n"); @@ -799,11 +900,6 @@ int rknpu_get_bw_priority(struct rknpu_device *rknpu_dev, uint32_t *priority, { void __iomem *base = rknpu_dev->bw_priority_base; - if (!rknpu_dev->config->bw_enable) { - LOG_WARN("Get bw_priority is not supported on this device!\n"); - return 0; - } - if (!base) return -EINVAL; @@ -828,11 +924,6 @@ int rknpu_set_bw_priority(struct rknpu_device *rknpu_dev, uint32_t priority, { void __iomem *base = rknpu_dev->bw_priority_base; - if (!rknpu_dev->config->bw_enable) { - LOG_WARN("Set bw_priority is not supported on this device!\n"); - return 0; - } - if (!base) return -EINVAL; @@ -855,28 +946,44 @@ int rknpu_set_bw_priority(struct rknpu_device *rknpu_dev, uint32_t priority, int rknpu_clear_rw_amount(struct rknpu_device *rknpu_dev) { void __iomem *rknpu_core_base = rknpu_dev->base[0]; + const struct rknpu_config *config = rknpu_dev->config; + unsigned long flags; - if (!rknpu_dev->config->bw_enable) { + if (config->amount_top == NULL) { LOG_WARN("Clear rw_amount is not supported on this device!\n"); return 0; } - spin_lock(&rknpu_dev->lock); + if (config->pc_dma_ctrl) { + uint32_t pc_data_addr = 0; - if (rknpu_dev->config->pc_dma_ctrl) { - uint32_t pc_data_addr = REG_READ(RKNPU_OFFSET_PC_DATA_ADDR); + spin_lock_irqsave(&rknpu_dev->irq_lock, flags); + pc_data_addr = REG_READ(RKNPU_OFFSET_PC_DATA_ADDR); REG_WRITE(0x1, RKNPU_OFFSET_PC_DATA_ADDR); - REG_WRITE(0x80000101, RKNPU_OFFSET_CLR_ALL_RW_AMOUNT); - REG_WRITE(0x00000101, RKNPU_OFFSET_CLR_ALL_RW_AMOUNT); + REG_WRITE(0x80000101, config->amount_top->offset_clr_all); + REG_WRITE(0x00000101, config->amount_top->offset_clr_all); + if (config->amount_core) { + REG_WRITE(0x80000101, + config->amount_core->offset_clr_all); + REG_WRITE(0x00000101, + config->amount_core->offset_clr_all); + } REG_WRITE(pc_data_addr, RKNPU_OFFSET_PC_DATA_ADDR); + spin_unlock_irqrestore(&rknpu_dev->irq_lock, flags); } else { - REG_WRITE(0x80000101, RKNPU_OFFSET_CLR_ALL_RW_AMOUNT); - REG_WRITE(0x00000101, RKNPU_OFFSET_CLR_ALL_RW_AMOUNT); + spin_lock(&rknpu_dev->lock); + REG_WRITE(0x80000101, config->amount_top->offset_clr_all); + REG_WRITE(0x00000101, config->amount_top->offset_clr_all); + if (config->amount_core) { + REG_WRITE(0x80000101, + config->amount_core->offset_clr_all); + REG_WRITE(0x00000101, + config->amount_core->offset_clr_all); + } + spin_unlock(&rknpu_dev->lock); } - spin_unlock(&rknpu_dev->lock); - return 0; } @@ -884,23 +991,42 @@ int rknpu_get_rw_amount(struct rknpu_device *rknpu_dev, uint32_t *dt_wr, uint32_t *dt_rd, uint32_t *wd_rd) { void __iomem *rknpu_core_base = rknpu_dev->base[0]; - int amount_scale = rknpu_dev->config->pc_data_amount_scale; + const struct rknpu_config *config = rknpu_dev->config; + int amount_scale = config->pc_data_amount_scale; - if (!rknpu_dev->config->bw_enable) { + if (config->amount_top == NULL) { LOG_WARN("Get rw_amount is not supported on this device!\n"); return 0; } spin_lock(&rknpu_dev->lock); - if (dt_wr != NULL) - *dt_wr = REG_READ(RKNPU_OFFSET_DT_WR_AMOUNT) * amount_scale; + if (dt_wr != NULL) { + *dt_wr = REG_READ(config->amount_top->offset_dt_wr) * + amount_scale; + if (config->amount_core) { + *dt_wr += REG_READ(config->amount_core->offset_dt_wr) * + amount_scale; + } + } - if (dt_rd != NULL) - *dt_rd = REG_READ(RKNPU_OFFSET_DT_RD_AMOUNT) * amount_scale; + if (dt_rd != NULL) { + *dt_rd = REG_READ(config->amount_top->offset_dt_rd) * + amount_scale; + if (config->amount_core) { + *dt_rd += REG_READ(config->amount_core->offset_dt_rd) * + amount_scale; + } + } - if (wd_rd != NULL) - *wd_rd = REG_READ(RKNPU_OFFSET_WT_RD_AMOUNT) * amount_scale; + if (wd_rd != NULL) { + *wd_rd = REG_READ(config->amount_top->offset_wt_rd) * + amount_scale; + if (config->amount_core) { + *wd_rd += REG_READ(config->amount_core->offset_wt_rd) * + amount_scale; + } + } spin_unlock(&rknpu_dev->lock); @@ -909,12 +1035,13 @@ int rknpu_get_rw_amount(struct rknpu_device *rknpu_dev, uint32_t *dt_wr, int rknpu_get_total_rw_amount(struct rknpu_device *rknpu_dev, uint32_t *amount) { + const struct rknpu_config *config = rknpu_dev->config; uint32_t dt_wr = 0; uint32_t dt_rd = 0; uint32_t wd_rd = 0; int ret = -EINVAL; - if (!rknpu_dev->config->bw_enable) { + if (config->amount_top == NULL) { LOG_WARN( "Get total_rw_amount is not supported on this device!\n"); return 0; diff --git a/target/linux/rockchip/rk35xx/files/drivers/rknpu/rknpu_mem.c b/target/linux/rockchip/rk35xx/files/drivers/rknpu/rknpu_mem.c index 4fd6866879..31aede9515 100644 --- a/target/linux/rockchip/rk35xx/files/drivers/rknpu/rknpu_mem.c +++ b/target/linux/rockchip/rk35xx/files/drivers/rknpu/rknpu_mem.c @@ -15,7 +15,10 @@ #include "rknpu_ioctl.h" #include "rknpu_mem.h" -int rknpu_mem_create_ioctl(struct rknpu_device *rknpu_dev, unsigned long data) +#ifdef CONFIG_ROCKCHIP_RKNPU_DMA_HEAP + +int rknpu_mem_create_ioctl(struct rknpu_device *rknpu_dev, struct file *file, + unsigned int cmd, unsigned long data) { struct rknpu_mem_create args; int ret = -EINVAL; @@ -27,16 +30,23 @@ int rknpu_mem_create_ioctl(struct rknpu_device *rknpu_dev, unsigned long data) struct page **pages; struct page *page; struct rknpu_mem_object *rknpu_obj = NULL; + struct rknpu_session *session = NULL; int i, fd; unsigned int length, page_count; + unsigned int in_size = _IOC_SIZE(cmd); + unsigned int k_size = sizeof(struct rknpu_mem_create); + char *k_data = (char *)&args; if (unlikely(copy_from_user(&args, (struct rknpu_mem_create *)data, - sizeof(struct rknpu_mem_create)))) { + in_size))) { LOG_ERROR("%s: copy_from_user failed\n", __func__); ret = -EFAULT; return ret; } + if (k_size > in_size) + memset(k_data + in_size, 0, k_size - in_size); + if (args.flags & RKNPU_MEM_NON_CONTIGUOUS) { LOG_ERROR("%s: malloc iommu memory unsupported in current!\n", __func__); @@ -65,6 +75,8 @@ int rknpu_mem_create_ioctl(struct rknpu_device *rknpu_dev, unsigned long data) O_CLOEXEC | O_RDWR, 0x0, dev_name(rknpu_dev->dev)); if (IS_ERR(dmabuf)) { + LOG_ERROR("dmabuf alloc failed, args.size = %llu\n", + args.size); ret = PTR_ERR(dmabuf); goto err_free_obj; } @@ -74,6 +86,7 @@ int rknpu_mem_create_ioctl(struct rknpu_device *rknpu_dev, unsigned long data) fd = dma_buf_fd(dmabuf, O_CLOEXEC | O_RDWR); if (fd < 0) { + LOG_ERROR("dmabuf fd get failed\n"); ret = -EFAULT; goto err_free_dma_buf; } @@ -81,12 +94,14 @@ int rknpu_mem_create_ioctl(struct rknpu_device *rknpu_dev, unsigned long data) attachment = dma_buf_attach(dmabuf, rknpu_dev->dev); if (IS_ERR(attachment)) { + LOG_ERROR("dma_buf_attach failed\n"); ret = PTR_ERR(attachment); goto err_free_dma_buf; } table = dma_buf_map_attachment(attachment, DMA_BIDIRECTIONAL); if (IS_ERR(table)) { + LOG_ERROR("dma_buf_attach failed\n"); dma_buf_detach(dmabuf, attachment); ret = PTR_ERR(table); goto err_free_dma_buf; @@ -100,20 +115,27 @@ int rknpu_mem_create_ioctl(struct rknpu_device *rknpu_dev, unsigned long data) __LINE__, &phys, length); } - page_count = length >> PAGE_SHIFT; - pages = kmalloc_array(page_count, sizeof(struct page), GFP_KERNEL); - if (!pages) { - ret = -ENOMEM; - goto err_detach_dma_buf; - } + if (args.flags & RKNPU_MEM_KERNEL_MAPPING) { + page_count = length >> PAGE_SHIFT; + pages = vmalloc(page_count * sizeof(struct page)); + if (!pages) { + LOG_ERROR("alloc pages failed\n"); + ret = -ENOMEM; + goto err_detach_dma_buf; + } - for (i = 0; i < page_count; i++) - pages[i] = &page[i]; + for (i = 0; i < page_count; i++) + pages[i] = &page[i]; - rknpu_obj->kv_addr = vmap(pages, page_count, VM_MAP, PAGE_KERNEL); - if (!rknpu_obj->kv_addr) { - ret = -ENOMEM; - goto err_free_pages; + rknpu_obj->kv_addr = + vmap(pages, page_count, VM_MAP, PAGE_KERNEL); + if (!rknpu_obj->kv_addr) { + LOG_ERROR("vmap pages addr failed\n"); + ret = -ENOMEM; + goto err_free_pages; + } + vfree(pages); + pages = NULL; } rknpu_obj->size = PAGE_ALIGN(args.size); @@ -131,16 +153,27 @@ int rknpu_mem_create_ioctl(struct rknpu_device *rknpu_dev, unsigned long data) (__u64)rknpu_obj->dma_addr); if (unlikely(copy_to_user((struct rknpu_mem_create *)data, &args, - sizeof(struct rknpu_mem_create)))) { + in_size))) { LOG_ERROR("%s: copy_to_user failed\n", __func__); ret = -EFAULT; goto err_unmap_kv_addr; } - kfree(pages); dma_buf_unmap_attachment(attachment, table, DMA_BIDIRECTIONAL); dma_buf_detach(dmabuf, attachment); + spin_lock(&rknpu_dev->lock); + + session = file->private_data; + if (!session) { + spin_unlock(&rknpu_dev->lock); + ret = -EFAULT; + goto err_unmap_kv_addr; + } + list_add_tail(&rknpu_obj->head, &session->list); + + spin_unlock(&rknpu_dev->lock); + return 0; err_unmap_kv_addr: @@ -148,7 +181,8 @@ err_unmap_kv_addr: rknpu_obj->kv_addr = NULL; err_free_pages: - kfree(pages); + vfree(pages); + pages = NULL; err_detach_dma_buf: dma_buf_unmap_attachment(attachment, table, DMA_BIDIRECTIONAL); @@ -166,11 +200,12 @@ err_free_obj: return ret; } -int rknpu_mem_destroy_ioctl(struct rknpu_device *rknpu_dev, unsigned long data) +int rknpu_mem_destroy_ioctl(struct rknpu_device *rknpu_dev, struct file *file, + unsigned long data) { - struct rknpu_mem_object *rknpu_obj = NULL; + struct rknpu_mem_object *rknpu_obj, *entry, *q; + struct rknpu_session *session = NULL; struct rknpu_mem_destroy args; - struct dma_buf *dmabuf; int ret = -EFAULT; if (unlikely(copy_from_user(&args, (struct rknpu_mem_destroy *)data, @@ -188,19 +223,35 @@ int rknpu_mem_destroy_ioctl(struct rknpu_device *rknpu_dev, unsigned long data) } rknpu_obj = (struct rknpu_mem_object *)(uintptr_t)args.obj_addr; - dmabuf = rknpu_obj->dmabuf; LOG_DEBUG( "free args.handle: %d, rknpu_obj: %#llx, rknpu_obj->dma_addr: %#llx\n", args.handle, (__u64)(uintptr_t)rknpu_obj, (__u64)rknpu_obj->dma_addr); - vunmap(rknpu_obj->kv_addr); - rknpu_obj->kv_addr = NULL; + spin_lock(&rknpu_dev->lock); + session = file->private_data; + if (!session) { + spin_unlock(&rknpu_dev->lock); + ret = -EFAULT; + return ret; + } + list_for_each_entry_safe(entry, q, &session->list, head) { + if (entry == rknpu_obj) { + list_del(&entry->head); + break; + } + } + spin_unlock(&rknpu_dev->lock); - if (!rknpu_obj->owner) - dma_buf_put(dmabuf); + if (rknpu_obj == entry) { + vunmap(rknpu_obj->kv_addr); + rknpu_obj->kv_addr = NULL; - kfree(rknpu_obj); + if (!rknpu_obj->owner) + dma_buf_put(rknpu_obj->dmabuf); + + kfree(rknpu_obj); + } return 0; } @@ -209,11 +260,9 @@ int rknpu_mem_destroy_ioctl(struct rknpu_device *rknpu_dev, unsigned long data) * begin cpu access => for_cpu = true * end cpu access => for_cpu = false */ -static void __maybe_unused rknpu_dma_buf_sync(struct rknpu_device *rknpu_dev, - struct rknpu_mem_object *rknpu_obj, - u32 offset, u32 length, - enum dma_data_direction dir, - bool for_cpu) +static void __maybe_unused rknpu_dma_buf_sync( + struct rknpu_device *rknpu_dev, struct rknpu_mem_object *rknpu_obj, + u32 offset, u32 length, enum dma_data_direction dir, bool for_cpu) { struct device *dev = rknpu_dev->dev; struct sg_table *sgt = rknpu_obj->sgt; @@ -256,7 +305,9 @@ int rknpu_mem_sync_ioctl(struct rknpu_device *rknpu_dev, unsigned long data) { struct rknpu_mem_object *rknpu_obj = NULL; struct rknpu_mem_sync args; +#ifdef CONFIG_DMABUF_PARTIAL struct dma_buf *dmabuf; +#endif int ret = -EFAULT; if (unlikely(copy_from_user(&args, (struct rknpu_mem_sync *)data, @@ -274,20 +325,18 @@ int rknpu_mem_sync_ioctl(struct rknpu_device *rknpu_dev, unsigned long data) } rknpu_obj = (struct rknpu_mem_object *)(uintptr_t)args.obj_addr; - dmabuf = rknpu_obj->dmabuf; #ifndef CONFIG_DMABUF_PARTIAL if (args.flags & RKNPU_MEM_SYNC_TO_DEVICE) { - rknpu_dma_buf_sync(rknpu_dev, rknpu_obj, - args.offset, args.size, DMA_TO_DEVICE, - false); + rknpu_dma_buf_sync(rknpu_dev, rknpu_obj, args.offset, args.size, + DMA_TO_DEVICE, false); } if (args.flags & RKNPU_MEM_SYNC_FROM_DEVICE) { - rknpu_dma_buf_sync(rknpu_dev, rknpu_obj, - args.offset, args.size, DMA_FROM_DEVICE, - true); + rknpu_dma_buf_sync(rknpu_dev, rknpu_obj, args.offset, args.size, + DMA_FROM_DEVICE, true); } #else + dmabuf = rknpu_obj->dmabuf; if (args.flags & RKNPU_MEM_SYNC_TO_DEVICE) { dmabuf->ops->end_cpu_access_partial(dmabuf, DMA_TO_DEVICE, args.offset, args.size); @@ -300,3 +349,5 @@ int rknpu_mem_sync_ioctl(struct rknpu_device *rknpu_dev, unsigned long data) return 0; } + +#endif diff --git a/target/linux/rockchip/rk35xx/files/drivers/rknpu/rknpu_mm.c b/target/linux/rockchip/rk35xx/files/drivers/rknpu/rknpu_mm.c index 9a13c3e256..a21bb6ded1 100644 --- a/target/linux/rockchip/rk35xx/files/drivers/rknpu/rknpu_mm.c +++ b/target/linux/rockchip/rk35xx/files/drivers/rknpu/rknpu_mm.c @@ -236,54 +236,3 @@ int rknpu_mm_dump(struct seq_file *m, void *data) return 0; } - -dma_addr_t rknpu_iommu_dma_alloc_iova(struct iommu_domain *domain, size_t size, - u64 dma_limit, struct device *dev) -{ - struct rknpu_iommu_dma_cookie *cookie = domain->iova_cookie; - struct iova_domain *iovad = &cookie->iovad; - unsigned long shift, iova_len, iova = 0; -#if (KERNEL_VERSION(5, 4, 0) > LINUX_VERSION_CODE) - dma_addr_t limit; -#endif - - shift = iova_shift(iovad); - iova_len = size >> shift; - /* - * Freeing non-power-of-two-sized allocations back into the IOVA caches - * will come back to bite us badly, so we have to waste a bit of space - * rounding up anything cacheable to make sure that can't happen. The - * order of the unadjusted size will still match upon freeing. - */ - if (iova_len < (1 << (IOVA_RANGE_CACHE_MAX_SIZE - 1))) - iova_len = roundup_pow_of_two(iova_len); - -#if (KERNEL_VERSION(5, 10, 0) <= LINUX_VERSION_CODE) - dma_limit = min_not_zero(dma_limit, dev->bus_dma_limit); -#else - if (dev->bus_dma_mask) - dma_limit &= dev->bus_dma_mask; -#endif - - if (domain->geometry.force_aperture) - dma_limit = - min_t(u64, dma_limit, domain->geometry.aperture_end); - -#if (KERNEL_VERSION(5, 4, 0) <= LINUX_VERSION_CODE) - iova = alloc_iova_fast(iovad, iova_len, dma_limit >> shift, true); -#else - limit = min_t(dma_addr_t, dma_limit >> shift, iovad->end_pfn); - - iova = alloc_iova_fast(iovad, iova_len, limit, true); -#endif - - return (dma_addr_t)iova << shift; -} - -void rknpu_iommu_dma_free_iova(struct rknpu_iommu_dma_cookie *cookie, - dma_addr_t iova, size_t size) -{ - struct iova_domain *iovad = &cookie->iovad; - - free_iova_fast(iovad, iova_pfn(iovad, iova), size >> iova_shift(iovad)); -} diff --git a/target/linux/rockchip/rk35xx/files/drivers/rknpu/rknpu_reset.c b/target/linux/rockchip/rk35xx/files/drivers/rknpu/rknpu_reset.c index 91c9b75d68..7bcf75b028 100644 --- a/target/linux/rockchip/rk35xx/files/drivers/rknpu/rknpu_reset.c +++ b/target/linux/rockchip/rk35xx/files/drivers/rknpu/rknpu_reset.c @@ -28,27 +28,34 @@ static inline struct reset_control *rknpu_reset_control_get(struct device *dev, int rknpu_reset_get(struct rknpu_device *rknpu_dev) { #ifndef FPGA_PLATFORM - struct reset_control *srst_a = NULL; - struct reset_control *srst_h = NULL; int i = 0; + int num_srsts = 0; - for (i = 0; i < rknpu_dev->config->num_resets; i++) { - srst_a = rknpu_reset_control_get( - rknpu_dev->dev, - rknpu_dev->config->resets[i].srst_a_name); - if (IS_ERR(srst_a)) - return PTR_ERR(srst_a); - - rknpu_dev->srst_a[i] = srst_a; - - srst_h = rknpu_reset_control_get( - rknpu_dev->dev, - rknpu_dev->config->resets[i].srst_h_name); - if (IS_ERR(srst_h)) - return PTR_ERR(srst_h); - - rknpu_dev->srst_h[i] = srst_h; + num_srsts = of_count_phandle_with_args(rknpu_dev->dev->of_node, + "resets", "#reset-cells"); + if (num_srsts <= 0) { + LOG_DEV_ERROR(rknpu_dev->dev, + "failed to get rknpu resets from dtb\n"); + return num_srsts; } + + rknpu_dev->srsts = devm_kcalloc(rknpu_dev->dev, num_srsts, + sizeof(*rknpu_dev->srsts), GFP_KERNEL); + if (!rknpu_dev->srsts) + return -ENOMEM; + + for (i = 0; i < num_srsts; ++i) { + rknpu_dev->srsts[i] = devm_reset_control_get_exclusive_by_index( + rknpu_dev->dev, i); + if (IS_ERR(rknpu_dev->srsts[i])) { + rknpu_dev->num_srsts = i; + return PTR_ERR(rknpu_dev->srsts[i]); + } + } + + rknpu_dev->num_srsts = num_srsts; + + return num_srsts; #endif return 0; @@ -93,7 +100,7 @@ int rknpu_soft_reset(struct rknpu_device *rknpu_dev) #ifndef FPGA_PLATFORM struct iommu_domain *domain = NULL; struct rknpu_subcore_data *subcore_data = NULL; - int ret = -EINVAL, i = 0; + int ret = 0, i = 0; if (rknpu_dev->bypass_soft_reset) { LOG_WARN("bypass soft reset\n"); @@ -112,17 +119,17 @@ int rknpu_soft_reset(struct rknpu_device *rknpu_dev) wake_up(&subcore_data->job_done_wq); } - LOG_INFO("soft reset\n"); + LOG_INFO("soft reset, num: %d\n", rknpu_dev->num_srsts); - for (i = 0; i < rknpu_dev->config->num_resets; i++) { - ret = rknpu_reset_assert(rknpu_dev->srst_a[i]); - ret |= rknpu_reset_assert(rknpu_dev->srst_h[i]); + for (i = 0; i < rknpu_dev->num_srsts; ++i) + ret |= rknpu_reset_assert(rknpu_dev->srsts[i]); - udelay(10); + udelay(10); - ret |= rknpu_reset_deassert(rknpu_dev->srst_a[i]); - ret |= rknpu_reset_deassert(rknpu_dev->srst_h[i]); - } + for (i = 0; i < rknpu_dev->num_srsts; ++i) + ret |= rknpu_reset_deassert(rknpu_dev->srsts[i]); + + udelay(10); if (ret) { LOG_DEV_ERROR(rknpu_dev->dev, @@ -141,6 +148,9 @@ int rknpu_soft_reset(struct rknpu_device *rknpu_dev) rknpu_dev->soft_reseting = false; + if (rknpu_dev->config->state_init != NULL) + rknpu_dev->config->state_init(rknpu_dev); + mutex_unlock(&rknpu_dev->reset_lock); #endif diff --git a/target/linux/rockchip/rk35xx/patches-5.10/1027-export-iommu_setup_dma_ops.patch b/target/linux/rockchip/rk35xx/patches-5.10/1027-export-iommu_setup_dma_ops.patch new file mode 100644 index 0000000000..c9fa98a72a --- /dev/null +++ b/target/linux/rockchip/rk35xx/patches-5.10/1027-export-iommu_setup_dma_ops.patch @@ -0,0 +1,24 @@ +From 2f679264ca2dc5e9b34e4690740490b52f1f21a8 Mon Sep 17 00:00:00 2001 +From: jjm2473 +Date: Wed, 12 Feb 2025 12:00:55 +0800 +Subject: [PATCH] dma-iommu: export iommu_setup_dma_ops + +--- + drivers/iommu/dma-iommu.c | 1 + + 1 file changed, 1 insertion(+) + +diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c +index b6452257d..10ed1b8ff 100644 +--- a/drivers/iommu/dma-iommu.c ++++ b/drivers/iommu/dma-iommu.c +@@ -1243,6 +1243,7 @@ void iommu_setup_dma_ops(struct device *dev, u64 dma_base, u64 size) + pr_warn("Failed to set up IOMMU for device %s; retaining platform DMA ops\n", + dev_name(dev)); + } ++EXPORT_SYMBOL_GPL(iommu_setup_dma_ops); + + static struct iommu_dma_msi_page *iommu_dma_get_msi_page(struct device *dev, + phys_addr_t msi_addr, struct iommu_domain *domain) +-- +2.46.0 + diff --git a/target/linux/rockchip/rk35xx/patches-5.10/1028-drivers-rknpu-backport.patch b/target/linux/rockchip/rk35xx/patches-5.10/1028-drivers-rknpu-backport.patch new file mode 100644 index 0000000000..71d6cf0c09 --- /dev/null +++ b/target/linux/rockchip/rk35xx/patches-5.10/1028-drivers-rknpu-backport.patch @@ -0,0 +1,206 @@ +From e16a5515f68cced6b8f54880bc3bd22702a0528a Mon Sep 17 00:00:00 2001 +From: jjm2473 +Date: Tue, 11 Feb 2025 13:51:29 +0800 +Subject: [PATCH] drivers/rknpu: backport to 5.10 + +--- + drivers/rknpu/rknpu_devfreq.c | 8 ++++---- + drivers/rknpu/rknpu_drv.c | 10 ++++++++++ + drivers/rknpu/rknpu_gem.c | 23 +++++++++++++++++++++++ + 3 files changed, 37 insertions(+), 4 deletions(-) + +diff --git a/drivers/rknpu/rknpu_devfreq.c b/drivers/rknpu/rknpu_devfreq.c +index 2668d2c46..ea717dde2 100644 +--- a/drivers/rknpu/rknpu_devfreq.c ++++ b/drivers/rknpu/rknpu_devfreq.c +@@ -19,7 +19,7 @@ static int npu_devfreq_target(struct device *dev, unsigned long *freq, + u32 flags); + + static struct monitor_dev_profile npu_mdevp = { +- .type = MONITOR_TYPE_DEV, ++ .type = MONITOR_TPYE_DEV, + .low_temp_adjust = rockchip_monitor_dev_low_temp_adjust, + .high_temp_adjust = rockchip_monitor_dev_high_temp_adjust, + #if KERNEL_VERSION(6, 1, 0) <= LINUX_VERSION_CODE +@@ -395,7 +395,7 @@ int rknpu_devfreq_init(struct rknpu_device *rknpu_dev) + err_remove_governor: + devfreq_remove_governor(&devfreq_rknpu_ondemand); + err_uinit_table: +-#if KERNEL_VERSION(5, 10, 198) <= LINUX_VERSION_CODE ++#if KERNEL_VERSION(6, 1, 0) <= LINUX_VERSION_CODE + rockchip_uninit_opp_table(dev, info); + #else + dev_pm_opp_of_remove_table(dev); +@@ -733,7 +733,7 @@ int rknpu_devfreq_init(struct rknpu_device *rknpu_dev) + err_remove_governor: + devfreq_remove_governor(&devfreq_rknpu_ondemand); + err_remove_table: +-#if KERNEL_VERSION(5, 10, 198) <= LINUX_VERSION_CODE ++#if KERNEL_VERSION(6, 1, 0) <= LINUX_VERSION_CODE + rockchip_uninit_opp_table(dev, &rknpu_dev->opp_info); + #else + dev_pm_opp_of_remove_table(dev); +@@ -798,7 +798,7 @@ void rknpu_devfreq_remove(struct rknpu_device *rknpu_dev) + } + if (rknpu_dev->devfreq) + devfreq_remove_governor(&devfreq_rknpu_ondemand); +-#if KERNEL_VERSION(5, 10, 198) <= LINUX_VERSION_CODE ++#if KERNEL_VERSION(6, 1, 0) <= LINUX_VERSION_CODE + rockchip_uninit_opp_table(rknpu_dev->dev, &rknpu_dev->opp_info); + #else + dev_pm_opp_of_remove_table(rknpu_dev->dev); +diff --git a/drivers/rknpu/rknpu_drv.c b/drivers/rknpu/rknpu_drv.c +index e8afa9778..25fa35aec 100644 +--- a/drivers/rknpu/rknpu_drv.c ++++ b/drivers/rknpu/rknpu_drv.c +@@ -108,6 +108,7 @@ static const struct rknpu_amount_data rknpu_core_amount = { + .offset_wt_rd = 0x243c, + }; + ++#ifdef CONFIG_CPU_RK3576 + static void rk3576_state_init(struct rknpu_device *rknpu_dev) + { + void __iomem *rknpu_core_base = rknpu_dev->base[0]; +@@ -158,6 +159,7 @@ static int rk3576_cache_sgt_init(struct rknpu_device *rknpu_dev) + + return ret; + } ++#endif + + static const struct rknpu_config rk356x_rknpu_config = { + .bw_priority_addr = 0xfe180008, +@@ -243,6 +245,7 @@ static const struct rknpu_config rv1106_rknpu_config = { + .cache_sgt_init = NULL, + }; + ++#ifdef CONFIG_CPU_RK3562 + static const struct rknpu_config rk3562_rknpu_config = { + .bw_priority_addr = 0x0, + .bw_priority_length = 0x0, +@@ -263,7 +266,9 @@ static const struct rknpu_config rk3562_rknpu_config = { + .state_init = NULL, + .cache_sgt_init = NULL, + }; ++#endif + ++#ifdef CONFIG_CPU_RK3576 + static const struct rknpu_config rk3576_rknpu_config = { + .bw_priority_addr = 0x0, + .bw_priority_length = 0x0, +@@ -284,6 +289,7 @@ static const struct rknpu_config rk3576_rknpu_config = { + .state_init = rk3576_state_init, + .cache_sgt_init = rk3576_cache_sgt_init, + }; ++#endif + + /* driver probe and init */ + static const struct of_device_id rknpu_of_match[] = { +@@ -303,14 +309,18 @@ static const struct of_device_id rknpu_of_match[] = { + .compatible = "rockchip,rv1106-rknpu", + .data = &rv1106_rknpu_config, + }, ++#ifdef CONFIG_CPU_RK3562 + { + .compatible = "rockchip,rk3562-rknpu", + .data = &rk3562_rknpu_config, + }, ++#endif ++#ifdef CONFIG_CPU_RK3576 + { + .compatible = "rockchip,rk3576-rknpu", + .data = &rk3576_rknpu_config, + }, ++#endif + {}, + }; + +diff --git a/drivers/rknpu/rknpu_gem.c b/drivers/rknpu/rknpu_gem.c +index a5c5354ed..14fdf643b 100644 +--- a/drivers/rknpu/rknpu_gem.c ++++ b/drivers/rknpu/rknpu_gem.c +@@ -28,6 +28,12 @@ + #include "rknpu_gem.h" + #include "rknpu_iommu.h" + ++#if IS_ENABLED(CONFIG_NO_GKI) && (IS_ENABLED(CONFIG_ROCKCHIP_RKNPU_SRAM) || defined(CONFIG_CPU_RK3562) || defined(CONFIG_CPU_RK3576)) ++#define RKNPU_CACHE_SYNC 1 ++#else ++#define RKNPU_CACHE_SYNC 0 ++#endif ++ + #define RKNPU_GEM_ALLOC_FROM_PAGES 1 + + #if RKNPU_GEM_ALLOC_FROM_PAGES +@@ -995,7 +1001,11 @@ static int rknpu_gem_mmap_pages(struct rknpu_gem_object *rknpu_obj, + struct drm_device *drm = rknpu_obj->base.dev; + int ret = -EINVAL; + ++#if KERNEL_VERSION(6, 3, 0) > LINUX_VERSION_CODE ++ vma->vm_flags |= VM_MIXEDMAP; ++#else + vm_flags_set(vma, VM_MIXEDMAP); ++#endif + + ret = __vm_map_pages(vma, rknpu_obj->pages, rknpu_obj->num_pages, + vma->vm_pgoff); +@@ -1088,7 +1098,11 @@ static int rknpu_gem_mmap_cache(struct rknpu_gem_object *rknpu_obj, + return -EINVAL; + } + ++#if KERNEL_VERSION(6, 3, 0) > LINUX_VERSION_CODE ++ vma->vm_flags |= VM_MIXEDMAP; ++#else + vm_flags_set(vma, VM_MIXEDMAP); ++#endif + + vm_size = vma->vm_end - vma->vm_start; + +@@ -1144,8 +1158,13 @@ static int rknpu_gem_mmap_buffer(struct rknpu_gem_object *rknpu_obj, + * vm_pgoff (used as a fake buffer offset by DRM) to 0 as we want to map + * the whole buffer. + */ ++#if KERNEL_VERSION(6, 3, 0) > LINUX_VERSION_CODE ++ vma->vm_flags |= VM_DONTCOPY | VM_DONTEXPAND | VM_DONTDUMP | VM_IO; ++ vma->vm_flags &= ~VM_PFNMAP; ++#else + vm_flags_set(vma, VM_DONTCOPY | VM_DONTEXPAND | VM_DONTDUMP | VM_IO); + vm_flags_clear(vma, VM_PFNMAP); ++#endif + vma->vm_pgoff = 0; + + vm_size = vma->vm_end - vma->vm_start; +@@ -1530,6 +1549,7 @@ int rknpu_gem_prime_mmap(struct drm_gem_object *obj, struct vm_area_struct *vma) + return rknpu_gem_mmap_obj(obj, vma); + } + ++#if RKNPU_CACHE_SYNC + static int rknpu_cache_sync_with_sg(struct rknpu_device *rknpu_dev, + struct rknpu_gem_object *rknpu_obj, + unsigned long *length, +@@ -1663,6 +1683,7 @@ static int rknpu_cache_sync(struct rknpu_gem_object *rknpu_obj, + + return 0; + } ++#endif + + int rknpu_gem_sync_ioctl(struct drm_device *dev, void *data, + struct drm_file *file_priv) +@@ -1708,6 +1729,7 @@ int rknpu_gem_sync_ioctl(struct drm_device *dev, void *data, + length = args->size; + offset = args->offset; + ++#if RKNPU_CACHE_SYNC + if (IS_ENABLED(CONFIG_NO_GKI) && + IS_ENABLED(CONFIG_ROCKCHIP_RKNPU_SRAM) && + rknpu_obj->sram_size > 0) { +@@ -1718,6 +1740,7 @@ int rknpu_gem_sync_ioctl(struct drm_device *dev, void *data, + rknpu_cache_sync(rknpu_obj, &length, &offset, + RKNPU_CACHE_NBUF, args->flags); + } ++#endif + + for_each_sg(rknpu_obj->sgt->sgl, sg, rknpu_obj->sgt->nents, i) { + if (length == 0) +-- +2.46.0 +