maple/istoreos
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

target/rk35xx: rknpu update to 0.9.8

Browse Source
This commit is contained in:
Liangbin Lian 2025-02-12 12:05:50 +08:00
parent 88450739b4
commit ce69c916b2
20 changed files with 3406 additions and 1333 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
target/linux/rockchip/rk35xx/files/drivers/rknpu/Makefile
View File

@ -9,6 +9,8 @@ rknpu-y += rknpu_drv.o
rknpu-y += rknpu_reset.o
rknpu-y += rknpu_job.o
rknpu-y += rknpu_debugger.o
rknpu-y += rknpu_iommu.o
rknpu-$(CONFIG_PM_DEVFREQ) += rknpu_devfreq.o
rknpu-$(CONFIG_ROCKCHIP_RKNPU_SRAM) += rknpu_mm.o
rknpu-$(CONFIG_ROCKCHIP_RKNPU_FENCE) += rknpu_fence.o
rknpu-$(CONFIG_ROCKCHIP_RKNPU_DRM_GEM) += rknpu_gem.o

46
target/linux/rockchip/rk35xx/files/drivers/rknpu/include/rknpu_devfreq.h Normal file
View File

@ -0,0 +1,46 @@
/* SPDX-License-Identifier: GPL-2.0 */
/*
* Copyright (C) Rockchip Electronics Co.Ltd
* Author: Finley Xiao <finley.xiao@rock-chips.com>
*/
#ifndef __LINUX_RKNPU_DEVFREQ_H
#define __LINUX_RKNPU_DEVFREQ_H
#ifdef CONFIG_PM_DEVFREQ
void rknpu_devfreq_lock(struct rknpu_device *rknpu_dev);
void rknpu_devfreq_unlock(struct rknpu_device *rknpu_dev);
int rknpu_devfreq_init(struct rknpu_device *rknpu_dev);
void rknpu_devfreq_remove(struct rknpu_device *rknpu_dev);
int rknpu_devfreq_runtime_suspend(struct device *dev);
int rknpu_devfreq_runtime_resume(struct device *dev);
#else
static inline int rknpu_devfreq_init(struct rknpu_device *rknpu_dev)
{
return -EOPNOTSUPP;
}
static inline void rknpu_devfreq_remove(struct rknpu_device *rknpu_dev)
{
}
static inline void rknpu_devfreq_lock(struct rknpu_device *rknpu_dev)
{
}
static inline void rknpu_devfreq_unlock(struct rknpu_device *rknpu_dev)
{
}
static inline int rknpu_devfreq_runtime_suspend(struct device *dev)
{
return -EOPNOTSUPP;
}
static inline int rknpu_devfreq_runtime_resume(struct device *dev)
{
return -EOPNOTSUPP;
}
#endif /* CONFIG_PM_DEVFREQ */
#endif /* __LINUX_RKNPU_DEVFREQ_H_ */

70
target/linux/rockchip/rk35xx/files/drivers/rknpu/include/rknpu_drv.h
View File

@ -10,6 +10,7 @@
#include <linux/completion.h>
#include <linux/device.h>
#include <linux/kref.h>
#include <linux/irq.h>
#include <linux/platform_device.h>
#include <linux/spinlock.h>
#include <linux/regulator/consumer.h>
@ -17,11 +18,9 @@
#include <linux/hrtimer.h>
#include <linux/miscdevice.h>
#ifndef FPGA_PLATFORM
#if KERNEL_VERSION(5, 10, 0) <= LINUX_VERSION_CODE
#include <soc/rockchip/rockchip_opp_select.h>
#endif
#endif
#include <soc/rockchip/rockchip_system_monitor.h>
#include <soc/rockchip/rockchip_ipa.h>
#include "rknpu_job.h"
#include "rknpu_fence.h"
@ -30,10 +29,10 @@
#define DRIVER_NAME "rknpu"
#define DRIVER_DESC "RKNPU driver"
#define DRIVER_DATE "20230202"
#define DRIVER_DATE "20240828"
#define DRIVER_MAJOR 0
#define DRIVER_MINOR 8
#define DRIVER_PATCHLEVEL 5
#define DRIVER_MINOR 9
#define DRIVER_PATCHLEVEL 8
#define LOG_TAG "RKNPU"
@ -54,9 +53,19 @@
#define LOG_DEV_DEBUG(dev, fmt, args...) dev_dbg(dev, LOG_TAG ": " fmt, ##args)
#define LOG_DEV_ERROR(dev, fmt, args...) dev_err(dev, LOG_TAG ": " fmt, ##args)
struct rknpu_reset_data {
const char *srst_a_name;
const char *srst_h_name;
#define RKNPU_MAX_IOMMU_DOMAIN_NUM 16
#define RKNPU_CACHE_SG_TABLE_NUM 2
struct rknpu_irqs_data {
const char *name;
irqreturn_t (*irq_hdl)(int irq, void *ctx);
};
struct rknpu_amount_data {
uint16_t offset_clr_all;
uint16_t offset_dt_wr;
uint16_t offset_dt_rd;
uint16_t offset_wt_rd;
};
struct rknpu_config {
@ -68,16 +77,21 @@ struct rknpu_config {
__u32 pc_task_number_mask;
__u32 pc_task_status_offset;
__u32 pc_dma_ctrl;
__u32 bw_enable;
const struct rknpu_irqs_data *irqs;
const struct rknpu_reset_data *resets;
int num_irqs;
int num_resets;
__u64 nbuf_phyaddr;
__u64 nbuf_size;
__u64 max_submit_number;
__u32 core_mask;
const struct rknpu_amount_data *amount_top;
const struct rknpu_amount_data *amount_core;
void (*state_init)(struct rknpu_device *rknpu_dev);
int (*cache_sgt_init)(struct rknpu_device *rknpu_dev);
};
struct rknpu_timer {
__u32 busy_time;
__u32 busy_time_record;
ktime_t busy_time;
ktime_t total_busy_time;
};
struct rknpu_subcore_data {
@ -99,6 +113,7 @@ struct rknpu_device {
void __iomem *base[RKNPU_MAX_CORES];
struct device *dev;
#ifdef CONFIG_ROCKCHIP_RKNPU_DRM_GEM
struct device *fake_dev;
struct drm_device *drm_dev;
#endif
#ifdef CONFIG_ROCKCHIP_RKNPU_DMA_HEAP
@ -110,13 +125,14 @@ struct rknpu_device {
spinlock_t irq_lock;
struct mutex power_lock;
struct mutex reset_lock;
struct mutex domain_lock;
struct rknpu_subcore_data subcore_datas[RKNPU_MAX_CORES];
const struct rknpu_config *config;
void __iomem *bw_priority_base;
struct rknpu_fence_context *fence_ctx;
bool iommu_en;
struct reset_control *srst_a[RKNPU_MAX_CORES];
struct reset_control *srst_h[RKNPU_MAX_CORES];
struct reset_control **srsts;
int num_srsts;
struct clk_bulk_data *clks;
int num_clks;
struct regulator *vdd;
@ -126,11 +142,7 @@ struct rknpu_device {
struct thermal_cooling_device *devfreq_cooling;
struct devfreq *devfreq;
unsigned long ondemand_freq;
#ifndef FPGA_PLATFORM
#if KERNEL_VERSION(5, 10, 0) <= LINUX_VERSION_CODE
struct rockchip_opp_info opp_info;
#endif
#endif
unsigned long current_freq;
unsigned long current_volt;
int bypass_irq_handler;
@ -149,13 +161,29 @@ struct rknpu_device {
ktime_t kt;
phys_addr_t sram_start;
phys_addr_t sram_end;
phys_addr_t nbuf_start;
phys_addr_t nbuf_end;
uint32_t sram_size;
uint32_t nbuf_size;
void __iomem *sram_base_io;
void __iomem *nbuf_base_io;
struct rknpu_mm *sram_mm;
unsigned long power_put_delay;
struct iommu_group *iommu_group;
int iommu_domain_num;
int iommu_domain_id;
struct iommu_domain *iommu_domains[RKNPU_MAX_IOMMU_DOMAIN_NUM];
struct sg_table *cache_sgt[RKNPU_CACHE_SG_TABLE_NUM];
atomic_t iommu_domain_refcount;
};
struct rknpu_session {
struct rknpu_device *rknpu_dev;
struct list_head list;
};
int rknpu_power_get(struct rknpu_device *rknpu_dev);
int rknpu_power_put(struct rknpu_device *rknpu_dev);
int rknpu_power_put_delay(struct rknpu_device *rknpu_dev);
#endif /* __LINUX_RKNPU_DRV_H_ */

24
target/linux/rockchip/rk35xx/files/drivers/rknpu/include/rknpu_gem.h
View File

@ -48,6 +48,7 @@ struct rknpu_gem_object {
unsigned int flags;
unsigned long size;
unsigned long sram_size;
unsigned long nbuf_size;
struct rknpu_mm_obj *sram_obj;
dma_addr_t iova_start;
unsigned long iova_size;
@ -59,13 +60,21 @@ struct rknpu_gem_object {
struct page **pages;
struct sg_table *sgt;
struct drm_mm_node mm_node;
int iommu_domain_id;
unsigned int core_mask;
unsigned int cache_with_sgt;
};
enum rknpu_cache_type {
RKNPU_CACHE_SRAM = 1 << 0,
RKNPU_CACHE_NBUF = 1 << 1,
};
/* create a new buffer with gem object */
struct rknpu_gem_object *rknpu_gem_object_create(struct drm_device *dev,
unsigned int flags,
unsigned long size,
unsigned long sram_size);
struct rknpu_gem_object *
rknpu_gem_object_create(struct drm_device *dev, unsigned int flags,
unsigned long size, unsigned long sram_size,
int iommu_domain_id, unsigned int core_mask);
/* destroy a buffer with gem object */
void rknpu_gem_object_destroy(struct rknpu_gem_object *rknpu_obj);
@ -157,6 +166,8 @@ int rknpu_gem_fault(struct vm_fault *vmf);
int rknpu_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf);
#endif
int rknpu_gem_mmap_obj(struct drm_gem_object *obj, struct vm_area_struct *vma);
/* set vm_flags and we can change the vm attribute to other one at here. */
int rknpu_gem_mmap(struct file *filp, struct vm_area_struct *vma);
@ -170,8 +181,13 @@ struct drm_gem_object *
rknpu_gem_prime_import_sg_table(struct drm_device *dev,
struct dma_buf_attachment *attach,
struct sg_table *sgt);
#if KERNEL_VERSION(6, 1, 0) > LINUX_VERSION_CODE
void *rknpu_gem_prime_vmap(struct drm_gem_object *obj);
void rknpu_gem_prime_vunmap(struct drm_gem_object *obj, void *vaddr);
#else
int rknpu_gem_prime_vmap(struct drm_gem_object *obj, struct iosys_map *map);
void rknpu_gem_prime_vunmap(struct drm_gem_object *obj, struct iosys_map *map);
#endif
int rknpu_gem_prime_mmap(struct drm_gem_object *obj,
struct vm_area_struct *vma);

62
target/linux/rockchip/rk35xx/files/drivers/rknpu/include/rknpu_ioctl.h
View File

@ -31,11 +31,6 @@
#define RKNPU_OFFSET_INT_STATUS 0x28
#define RKNPU_OFFSET_INT_RAW_STATUS 0x2c
#define RKNPU_OFFSET_CLR_ALL_RW_AMOUNT 0x8010
#define RKNPU_OFFSET_DT_WR_AMOUNT 0x8034
#define RKNPU_OFFSET_DT_RD_AMOUNT 0x8038
#define RKNPU_OFFSET_WT_RD_AMOUNT 0x803c
#define RKNPU_OFFSET_ENABLE_MASK 0xf008
#define RKNPU_INT_CLEAR 0x1ffff
@ -44,10 +39,10 @@
#define RKNPU_STR_HELPER(x) #x
#define RKNPU_GET_DRV_VERSION_STRING(MAJOR, MINOR, PATCHLEVEL) \
RKNPU_STR_HELPER(MAJOR) \
#define RKNPU_GET_DRV_VERSION_STRING(MAJOR, MINOR, PATCHLEVEL) \
RKNPU_STR_HELPER(MAJOR) \
"." RKNPU_STR_HELPER(MINOR) "." RKNPU_STR_HELPER(PATCHLEVEL)
#define RKNPU_GET_DRV_VERSION_CODE(MAJOR, MINOR, PATCHLEVEL) \
#define RKNPU_GET_DRV_VERSION_CODE(MAJOR, MINOR, PATCHLEVEL) \
(MAJOR * 10000 + MINOR * 100 + PATCHLEVEL)
#define RKNPU_GET_DRV_VERSION_MAJOR(CODE) (CODE / 10000)
#define RKNPU_GET_DRV_VERSION_MINOR(CODE) ((CODE % 10000) / 100)
@ -67,29 +62,34 @@ enum e_rknpu_mem_type {
RKNPU_MEM_WRITE_COMBINE = 1 << 2,
/* dma attr kernel mapping */
RKNPU_MEM_KERNEL_MAPPING = 1 << 3,
/* iommu mapping */
/* IOMMU mapping */
RKNPU_MEM_IOMMU = 1 << 4,
/* zero mapping */
RKNPU_MEM_ZEROING = 1 << 5,
/* allocate secure buffer */
RKNPU_MEM_SECURE = 1 << 6,
/* allocate from non-dma32 zone */
RKNPU_MEM_NON_DMA32 = 1 << 7,
/* allocate from dma32 zone */
RKNPU_MEM_DMA32 = 1 << 7,
/* request SRAM */
RKNPU_MEM_TRY_ALLOC_SRAM = 1 << 8,
/* request NBUF */
RKNPU_MEM_TRY_ALLOC_NBUF = 1 << 9,
/* IOMMU limiting IOVA alignment */
RKNPU_MEM_IOMMU_LIMIT_IOVA_ALIGNMENT = 1 << 10,
RKNPU_MEM_MASK = RKNPU_MEM_NON_CONTIGUOUS | RKNPU_MEM_CACHEABLE |
RKNPU_MEM_WRITE_COMBINE | RKNPU_MEM_KERNEL_MAPPING |
RKNPU_MEM_IOMMU | RKNPU_MEM_ZEROING |
RKNPU_MEM_SECURE | RKNPU_MEM_NON_DMA32 |
RKNPU_MEM_TRY_ALLOC_SRAM
RKNPU_MEM_SECURE | RKNPU_MEM_DMA32 |
RKNPU_MEM_TRY_ALLOC_SRAM | RKNPU_MEM_TRY_ALLOC_NBUF |
RKNPU_MEM_IOMMU_LIMIT_IOVA_ALIGNMENT
};
/* sync mode definitions. */
enum e_rknpu_mem_sync_mode {
RKNPU_MEM_SYNC_TO_DEVICE = 1 << 0,
RKNPU_MEM_SYNC_FROM_DEVICE = 1 << 1,
RKNPU_MEM_SYNC_MASK =
RKNPU_MEM_SYNC_TO_DEVICE | RKNPU_MEM_SYNC_FROM_DEVICE
RKNPU_MEM_SYNC_MASK = RKNPU_MEM_SYNC_TO_DEVICE |
RKNPU_MEM_SYNC_FROM_DEVICE
};
/* job mode definitions. */
@ -132,6 +132,8 @@ enum e_rknpu_action {
RKNPU_POWER_OFF = 21,
RKNPU_GET_TOTAL_SRAM_SIZE = 22,
RKNPU_GET_FREE_SRAM_SIZE = 23,
RKNPU_GET_IOMMU_DOMAIN_ID = 24,
RKNPU_SET_IOMMU_DOMAIN_ID = 25,
};
/**
@ -145,6 +147,8 @@ enum e_rknpu_action {
* @dma_addr: dma address that access by rknpu.
* @sram_size: user-desired sram memory allocation size.
* - this size value would be page-aligned internally.
* @iommu_domain_id: iommu domain id
* @reserved: just padding to be 64-bit aligned.
*/
struct rknpu_mem_create {
__u32 handle;
@ -153,6 +157,8 @@ struct rknpu_mem_create {
__u64 obj_addr;
__u64 dma_addr;
__u64 sram_size;
__s32 iommu_domain_id;
__u32 core_mask;
};
/**
@ -247,9 +253,10 @@ struct rknpu_subcore_task {
* @task_counter: task counter
* @priority: submit priority
* @task_obj_addr: address of task object
* @regcfg_obj_addr: address of register config object
* @iommu_domain_id: iommu domain id
* @reserved: just padding to be 64-bit aligned.
* @task_base_addr: task base address
* @user_data: (optional) user data
* @hw_elapse_time: hardware elapse time
* @core_mask: core mask of rknpu
* @fence_fd: dma fence fd
* @subcore_task: subcore task
@ -263,9 +270,10 @@ struct rknpu_submit {
__u32 task_counter;
__s32 priority;
__u64 task_obj_addr;
__u64 regcfg_obj_addr;
__u32 iommu_domain_id;
__u32 reserved;
__u64 task_base_addr;
__u64 user_data;
__s64 hw_elapse_time;
__u32 core_mask;
__s32 fence_fd;
struct rknpu_subcore_task subcore_task[5];
@ -297,25 +305,25 @@ struct rknpu_action {
#include <drm/drm.h>
#define DRM_IOCTL_RKNPU_ACTION \
#define DRM_IOCTL_RKNPU_ACTION \
DRM_IOWR(DRM_COMMAND_BASE + RKNPU_ACTION, struct rknpu_action)
#define DRM_IOCTL_RKNPU_SUBMIT \
#define DRM_IOCTL_RKNPU_SUBMIT \
DRM_IOWR(DRM_COMMAND_BASE + RKNPU_SUBMIT, struct rknpu_submit)
#define DRM_IOCTL_RKNPU_MEM_CREATE \
#define DRM_IOCTL_RKNPU_MEM_CREATE \
DRM_IOWR(DRM_COMMAND_BASE + RKNPU_MEM_CREATE, struct rknpu_mem_create)
#define DRM_IOCTL_RKNPU_MEM_MAP \
#define DRM_IOCTL_RKNPU_MEM_MAP \
DRM_IOWR(DRM_COMMAND_BASE + RKNPU_MEM_MAP, struct rknpu_mem_map)
#define DRM_IOCTL_RKNPU_MEM_DESTROY \
#define DRM_IOCTL_RKNPU_MEM_DESTROY \
DRM_IOWR(DRM_COMMAND_BASE + RKNPU_MEM_DESTROY, struct rknpu_mem_destroy)
#define DRM_IOCTL_RKNPU_MEM_SYNC \
#define DRM_IOCTL_RKNPU_MEM_SYNC \
DRM_IOWR(DRM_COMMAND_BASE + RKNPU_MEM_SYNC, struct rknpu_mem_sync)
#define IOCTL_RKNPU_ACTION RKNPU_IOWR(RKNPU_ACTION, struct rknpu_action)
#define IOCTL_RKNPU_SUBMIT RKNPU_IOWR(RKNPU_SUBMIT, struct rknpu_submit)
#define IOCTL_RKNPU_MEM_CREATE \
#define IOCTL_RKNPU_MEM_CREATE \
RKNPU_IOWR(RKNPU_MEM_CREATE, struct rknpu_mem_create)
#define IOCTL_RKNPU_MEM_MAP RKNPU_IOWR(RKNPU_MEM_MAP, struct rknpu_mem_map)
#define IOCTL_RKNPU_MEM_DESTROY \
#define IOCTL_RKNPU_MEM_DESTROY \
RKNPU_IOWR(RKNPU_MEM_DESTROY, struct rknpu_mem_destroy)
#define IOCTL_RKNPU_MEM_SYNC RKNPU_IOWR(RKNPU_MEM_SYNC, struct rknpu_mem_sync)

60
target/linux/rockchip/rk35xx/files/drivers/rknpu/include/rknpu_iommu.h Normal file
View File

@ -0,0 +1,60 @@
/* SPDX-License-Identifier: GPL-2.0 */
/*
* Copyright (C) Rockchip Electronics Co.Ltd
* Author: Felix Zeng <felix.zeng@rock-chips.com>
*/
#ifndef __LINUX_RKNPU_IOMMU_H
#define __LINUX_RKNPU_IOMMU_H
#include <linux/mutex.h>
#include <linux/seq_file.h>
#include <linux/iommu.h>
#include <linux/iova.h>
#include <linux/version.h>
#if KERNEL_VERSION(6, 1, 0) > LINUX_VERSION_CODE
#include <linux/dma-iommu.h>
#endif
#include "rknpu_drv.h"
enum iommu_dma_cookie_type {
IOMMU_DMA_IOVA_COOKIE,
IOMMU_DMA_MSI_COOKIE,
};
struct rknpu_iommu_dma_cookie {
enum iommu_dma_cookie_type type;
/* Full allocator for IOMMU_DMA_IOVA_COOKIE */
struct iova_domain iovad;
};
dma_addr_t rknpu_iommu_dma_alloc_iova(struct iommu_domain *domain, size_t size,
u64 dma_limit, struct device *dev,
bool size_aligned);
void rknpu_iommu_dma_free_iova(struct rknpu_iommu_dma_cookie *cookie,
dma_addr_t iova, size_t size, bool size_aligned);
int rknpu_iommu_dma_map_sg(struct device *dev, struct scatterlist *sg,
int nents, enum dma_data_direction dir,
bool iova_aligned);
void rknpu_iommu_dma_unmap_sg(struct device *dev, struct scatterlist *sg,
int nents, enum dma_data_direction dir,
bool iova_aligned);
int rknpu_iommu_init_domain(struct rknpu_device *rknpu_dev);
int rknpu_iommu_switch_domain(struct rknpu_device *rknpu_dev, int domain_id);
void rknpu_iommu_free_domains(struct rknpu_device *rknpu_dev);
int rknpu_iommu_domain_get_and_switch(struct rknpu_device *rknpu_dev,
int domain_id);
int rknpu_iommu_domain_put(struct rknpu_device *rknpu_dev);
#if KERNEL_VERSION(5, 10, 0) < LINUX_VERSION_CODE
int iommu_get_dma_cookie(struct iommu_domain *domain);
#endif
#endif

9
target/linux/rockchip/rk35xx/files/drivers/rknpu/include/rknpu_job.h
View File

@ -30,7 +30,6 @@ struct rknpu_job {
struct rknpu_device *rknpu_dev;
struct list_head head[RKNPU_MAX_CORES];
struct work_struct cleanup_work;
bool in_queue[RKNPU_MAX_CORES];
bool irq_entry[RKNPU_MAX_CORES];
unsigned int flags;
int ret;
@ -43,9 +42,13 @@ struct rknpu_job {
struct dma_fence *fence;
ktime_t timestamp;
uint32_t use_core_num;
uint32_t run_count;
uint32_t interrupt_count;
atomic_t run_count;
atomic_t interrupt_count;
ktime_t hw_commit_time;
ktime_t hw_recoder_time;
ktime_t hw_elapse_time;
atomic_t submit_count[RKNPU_MAX_CORES];
int iommu_domain_id;
};
irqreturn_t rknpu_core0_irq_handler(int irq, void *data);

7
target/linux/rockchip/rk35xx/files/drivers/rknpu/include/rknpu_mem.h
View File

@ -33,11 +33,14 @@ struct rknpu_mem_object {
struct page **pages;
struct sg_table *sgt;
struct dma_buf *dmabuf;
struct list_head head;
unsigned int owner;
};
int rknpu_mem_create_ioctl(struct rknpu_device *rknpu_dev, unsigned long data);
int rknpu_mem_destroy_ioctl(struct rknpu_device *rknpu_dev, unsigned long data);
int rknpu_mem_create_ioctl(struct rknpu_device *rknpu_dev, struct file *file,
unsigned int cmd, unsigned long data);
int rknpu_mem_destroy_ioctl(struct rknpu_device *rknpu_dev, struct file *file,
unsigned long data);
int rknpu_mem_sync_ioctl(struct rknpu_device *rknpu_dev, unsigned long data);
#endif

19
target/linux/rockchip/rk35xx/files/drivers/rknpu/include/rknpu_mm.h
View File

@ -10,7 +10,6 @@
#include <linux/mutex.h>
#include <linux/seq_file.h>
#include <linux/iommu.h>
#include <linux/dma-iommu.h>
#include <linux/iova.h>
#include "rknpu_drv.h"
@ -40,22 +39,4 @@ int rknpu_mm_free(struct rknpu_mm *mm, struct rknpu_mm_obj *mm_obj);
int rknpu_mm_dump(struct seq_file *m, void *data);
enum iommu_dma_cookie_type {
IOMMU_DMA_IOVA_COOKIE,
IOMMU_DMA_MSI_COOKIE,
};
struct rknpu_iommu_dma_cookie {
enum iommu_dma_cookie_type type;
/* Full allocator for IOMMU_DMA_IOVA_COOKIE */
struct iova_domain iovad;
};
dma_addr_t rknpu_iommu_dma_alloc_iova(struct iommu_domain *domain, size_t size,
u64 dma_limit, struct device *dev);
void rknpu_iommu_dma_free_iova(struct rknpu_iommu_dma_cookie *cookie,
dma_addr_t iova, size_t size);
#endif

14
target/linux/rockchip/rk35xx/files/drivers/rknpu/rknpu_debugger.c
View File

@ -46,7 +46,7 @@ static int rknpu_load_show(struct seq_file *m, void *data)
unsigned long flags;
int i;
int load;
uint64_t busy_time_total, div_value;
uint64_t total_busy_time, div_value;
seq_puts(m, "NPU load: ");
for (i = 0; i < rknpu_dev->config->num_irqs; i++) {
@ -57,13 +57,13 @@ static int rknpu_load_show(struct seq_file *m, void *data)
spin_lock_irqsave(&rknpu_dev->irq_lock, flags);
busy_time_total = subcore_data->timer.busy_time_record;
total_busy_time = subcore_data->timer.total_busy_time;
spin_unlock_irqrestore(&rknpu_dev->irq_lock, flags);
div_value = (RKNPU_LOAD_INTERVAL / 100000);
do_div(busy_time_total, div_value);
load = busy_time_total;
div_value = (RKNPU_LOAD_INTERVAL / 100);
do_div(total_busy_time, div_value);
load = total_busy_time > 100 ? 100 : total_busy_time;
if (rknpu_dev->config->num_irqs > 1)
seq_printf(m, "%2.d%%,", load);
@ -457,7 +457,11 @@ CREATE_FAIL:
#ifdef CONFIG_ROCKCHIP_RKNPU_PROC_FS
static int rknpu_procfs_open(struct inode *inode, struct file *file)
{
#if KERNEL_VERSION(6, 1, 0) > LINUX_VERSION_CODE
struct rknpu_debugger_node *node = PDE_DATA(inode);
#else
struct rknpu_debugger_node *node = pde_data(inode);
#endif
return single_open(file, node->info_ent->show, node);
}

807
target/linux/rockchip/rk35xx/files/drivers/rknpu/rknpu_devfreq.c Normal file
View File

@ -0,0 +1,807 @@
// SPDX-License-Identifier: GPL-2.0
/*
* Copyright (C) Rockchip Electronics Co.Ltd
* Author: Finley Xiao <finley.xiao@rock-chips.com>
*/
#include <linux/clk.h>
#include <linux/clk-provider.h>
#include <linux/devfreq_cooling.h>
#include <linux/pm_runtime.h>
#include <linux/regmap.h>
#include <../drivers/devfreq/governor.h>
#include "rknpu_drv.h"
#include "rknpu_devfreq.h"
#define POWER_DOWN_FREQ 200000000
static int npu_devfreq_target(struct device *dev, unsigned long *freq,
u32 flags);
static struct monitor_dev_profile npu_mdevp = {
.type = MONITOR_TYPE_DEV,
.low_temp_adjust = rockchip_monitor_dev_low_temp_adjust,
.high_temp_adjust = rockchip_monitor_dev_high_temp_adjust,
#if KERNEL_VERSION(6, 1, 0) <= LINUX_VERSION_CODE
.check_rate_volt = rockchip_monitor_check_rate_volt,
#else
.update_volt = rockchip_monitor_check_rate_volt,
#endif
};
static int npu_devfreq_get_dev_status(struct device *dev,
struct devfreq_dev_status *stat)
{
return 0;
}
static int npu_devfreq_get_cur_freq(struct device *dev, unsigned long *freq)
{
struct rknpu_device *rknpu_dev = dev_get_drvdata(dev);
*freq = rknpu_dev->current_freq;
return 0;
}
static struct devfreq_dev_profile npu_devfreq_profile = {
.polling_ms = 50,
.target = npu_devfreq_target,
.get_dev_status = npu_devfreq_get_dev_status,
.get_cur_freq = npu_devfreq_get_cur_freq,
};
static int devfreq_rknpu_ondemand_func(struct devfreq *df, unsigned long *freq)
{
struct rknpu_device *rknpu_dev = df->data;
if (rknpu_dev && rknpu_dev->ondemand_freq)
*freq = rknpu_dev->ondemand_freq;
else
*freq = df->previous_freq;
return 0;
}
static int devfreq_rknpu_ondemand_handler(struct devfreq *devfreq,
unsigned int event, void *data)
{
return 0;
}
static struct devfreq_governor devfreq_rknpu_ondemand = {
.name = "rknpu_ondemand",
.get_target_freq = devfreq_rknpu_ondemand_func,
.event_handler = devfreq_rknpu_ondemand_handler,
};
static int rk3576_npu_set_read_margin(struct device *dev,
struct rockchip_opp_info *opp_info,
u32 rm)
{
if (!opp_info->grf || !opp_info->volt_rm_tbl)
return 0;
if (rm == opp_info->current_rm || rm == UINT_MAX)
return 0;
LOG_DEV_DEBUG(dev, "set rm to %d\n", rm);
regmap_write(opp_info->grf, 0x08, 0x001c0000 | (rm << 2));
regmap_write(opp_info->grf, 0x0c, 0x003c0000 | (rm << 2));
regmap_write(opp_info->grf, 0x10, 0x001c0000 | (rm << 2));
return 0;
}
static int rk3588_npu_get_soc_info(struct device *dev, struct device_node *np,
int *bin, int *process)
{
int ret = 0;
u8 value = 0;
if (!bin)
return 0;
if (of_property_match_string(np, "nvmem-cell-names",
"specification_serial_number") >= 0) {
ret = rockchip_nvmem_cell_read_u8(
np, "specification_serial_number", &value);
if (ret) {
LOG_DEV_ERROR(
dev,
"Failed to get specification_serial_number\n");
return ret;
}
/* RK3588M */
if (value == 0xd)
*bin = 1;
/* RK3588J */
else if (value == 0xa)
*bin = 2;
}
if (*bin < 0)
*bin = 0;
LOG_DEV_INFO(dev, "bin=%d\n", *bin);
return ret;
}
#if KERNEL_VERSION(6, 1, 0) <= LINUX_VERSION_CODE
static int rk3588_npu_set_soc_info(struct device *dev, struct device_node *np,
struct rockchip_opp_info *opp_info)
{
int bin = opp_info->bin;
if (opp_info->volt_sel < 0)
return 0;
if (bin < 0)
bin = 0;
if (!of_property_read_bool(np, "rockchip,supported-hw"))
return 0;
/* SoC Version */
opp_info->supported_hw[0] = BIT(bin);
/* Speed Grade */
opp_info->supported_hw[1] = BIT(opp_info->volt_sel);
return 0;
}
#else
static int rk3588_npu_set_soc_info(struct device *dev, struct device_node *np,
int bin, int process, int volt_sel)
{
struct opp_table *opp_table;
u32 supported_hw[2];
if (volt_sel < 0)
return 0;
if (bin < 0)
bin = 0;
if (!of_property_read_bool(np, "rockchip,supported-hw"))
return 0;
/* SoC Version */
supported_hw[0] = BIT(bin);
/* Speed Grade */
supported_hw[1] = BIT(volt_sel);
opp_table = dev_pm_opp_set_supported_hw(dev, supported_hw, 2);
if (IS_ERR(opp_table)) {
LOG_DEV_ERROR(dev, "failed to set supported opp\n");
return PTR_ERR(opp_table);
}
return 0;
}
#endif
static int rk3588_npu_set_read_margin(struct device *dev,
struct rockchip_opp_info *opp_info,
u32 rm)
{
u32 offset = 0, val = 0;
int i, ret = 0;
if (!opp_info->grf || !opp_info->volt_rm_tbl)
return 0;
if (rm == opp_info->current_rm || rm == UINT_MAX)
return 0;
LOG_DEV_DEBUG(dev, "set rm to %d\n", rm);
for (i = 0; i < 3; i++) {
ret = regmap_read(opp_info->grf, offset, &val);
if (ret < 0) {
LOG_DEV_ERROR(dev, "failed to get rm from 0x%x\n",
offset);
return ret;
}
val &= ~0x1c;
regmap_write(opp_info->grf, offset, val | (rm << 2));
offset += 4;
}
opp_info->current_rm = rm;
return 0;
}
#if KERNEL_VERSION(6, 1, 0) <= LINUX_VERSION_CODE
static int npu_opp_config_regulators(struct device *dev,
struct dev_pm_opp *old_opp,
struct dev_pm_opp *new_opp,
struct regulator **regulators,
unsigned int count)
{
struct rknpu_device *rknpu_dev = dev_get_drvdata(dev);
return rockchip_opp_config_regulators(dev, old_opp, new_opp, regulators,
count, &rknpu_dev->opp_info);
}
static int npu_opp_config_clks(struct device *dev, struct opp_table *opp_table,
struct dev_pm_opp *opp, void *data,
bool scaling_down)
{
struct rknpu_device *rknpu_dev = dev_get_drvdata(dev);
return rockchip_opp_config_clks(dev, opp_table, opp, data, scaling_down,
&rknpu_dev->opp_info);
}
#endif
static const struct rockchip_opp_data rk3576_npu_opp_data = {
.set_read_margin = rk3576_npu_set_read_margin,
#if KERNEL_VERSION(6, 1, 0) <= LINUX_VERSION_CODE
.config_regulators = npu_opp_config_regulators,
.config_clks = npu_opp_config_clks,
#endif
};
static const struct rockchip_opp_data rk3588_npu_opp_data = {
.get_soc_info = rk3588_npu_get_soc_info,
.set_soc_info = rk3588_npu_set_soc_info,
.set_read_margin = rk3588_npu_set_read_margin,
#if KERNEL_VERSION(6, 1, 0) <= LINUX_VERSION_CODE
.config_regulators = npu_opp_config_regulators,
.config_clks = npu_opp_config_clks,
#endif
};
static const struct of_device_id rockchip_npu_of_match[] = {
{
.compatible = "rockchip,rk3576",
.data = (void *)&rk3576_npu_opp_data,
},
{
.compatible = "rockchip,rk3588",
.data = (void *)&rk3588_npu_opp_data,
},
{},
};
#if KERNEL_VERSION(6, 1, 0) <= LINUX_VERSION_CODE
void rknpu_devfreq_lock(struct rknpu_device *rknpu_dev)
{
if (rknpu_dev->devfreq)
rockchip_opp_dvfs_lock(&rknpu_dev->opp_info);
}
EXPORT_SYMBOL(rknpu_devfreq_lock);
void rknpu_devfreq_unlock(struct rknpu_device *rknpu_dev)
{
if (rknpu_dev->devfreq)
rockchip_opp_dvfs_unlock(&rknpu_dev->opp_info);
}
EXPORT_SYMBOL(rknpu_devfreq_unlock);
static int npu_devfreq_target(struct device *dev, unsigned long *freq,
u32 flags)
{
struct rknpu_device *rknpu_dev = dev_get_drvdata(dev);
struct rockchip_opp_info *opp_info = &rknpu_dev->opp_info;
struct dev_pm_opp *opp;
unsigned long opp_volt;
int ret = 0;
if (!opp_info->is_rate_volt_checked)
return -EINVAL;
opp = devfreq_recommended_opp(dev, freq, flags);
if (IS_ERR(opp))
return PTR_ERR(opp);
opp_volt = dev_pm_opp_get_voltage(opp);
dev_pm_opp_put(opp);
if (*freq == rknpu_dev->current_freq)
return 0;
rockchip_opp_dvfs_lock(opp_info);
if (pm_runtime_active(dev))
opp_info->is_runtime_active = true;
else
opp_info->is_runtime_active = false;
ret = dev_pm_opp_set_rate(dev, *freq);
if (!ret) {
rknpu_dev->current_freq = *freq;
if (rknpu_dev->devfreq)
rknpu_dev->devfreq->last_status.current_frequency =
*freq;
rknpu_dev->current_volt = opp_volt;
LOG_DEV_DEBUG(dev, "set rknpu freq: %lu, volt: %lu\n",
rknpu_dev->current_freq, rknpu_dev->current_volt);
}
rockchip_opp_dvfs_unlock(opp_info);
return ret;
}
static const struct rockchip_opp_data rockchip_npu_opp_data = {
.config_clks = npu_opp_config_clks,
};
int rknpu_devfreq_init(struct rknpu_device *rknpu_dev)
{
struct rockchip_opp_info *info = &rknpu_dev->opp_info;
struct device *dev = rknpu_dev->dev;
struct devfreq_dev_profile *dp;
struct dev_pm_opp *opp;
unsigned int dyn_power_coeff = 0;
int ret = 0;
info->data = &rockchip_npu_opp_data;
rockchip_get_opp_data(rockchip_npu_of_match, info);
ret = rockchip_init_opp_table(dev, info, "clk_npu", "rknpu");
if (ret) {
LOG_DEV_ERROR(dev, "failed to init_opp_table\n");
return -EINVAL;
}
rknpu_dev->current_freq = clk_get_rate(rknpu_dev->clks[0].clk);
opp = devfreq_recommended_opp(dev, &rknpu_dev->current_freq, 0);
if (IS_ERR(opp)) {
ret = PTR_ERR(opp);
goto err_uinit_table;
}
dev_pm_opp_put(opp);
dp = &npu_devfreq_profile;
dp->initial_freq = rknpu_dev->current_freq;
of_property_read_u32(dev->of_node, "dynamic-power-coefficient",
&dyn_power_coeff);
if (dyn_power_coeff)
dp->is_cooling_device = true;
ret = devfreq_add_governor(&devfreq_rknpu_ondemand);
if (ret) {
LOG_DEV_ERROR(dev, "failed to add rknpu_ondemand governor\n");
goto err_uinit_table;
}
rknpu_dev->devfreq = devm_devfreq_add_device(dev, dp, "rknpu_ondemand",
(void *)rknpu_dev);
if (IS_ERR(rknpu_dev->devfreq)) {
LOG_DEV_ERROR(dev, "failed to add devfreq\n");
ret = PTR_ERR(rknpu_dev->devfreq);
rknpu_dev->devfreq = NULL;
goto err_remove_governor;
}
npu_mdevp.data = rknpu_dev->devfreq;
npu_mdevp.opp_info = &rknpu_dev->opp_info;
rknpu_dev->mdev_info =
rockchip_system_monitor_register(dev, &npu_mdevp);
if (IS_ERR(rknpu_dev->mdev_info)) {
dev_dbg(dev, "without system monitor\n");
rknpu_dev->mdev_info = NULL;
}
rknpu_dev->current_freq = clk_get_rate(rknpu_dev->clks[0].clk);
rknpu_dev->ondemand_freq = rknpu_dev->current_freq;
rknpu_dev->current_volt = regulator_get_voltage(rknpu_dev->vdd);
rknpu_dev->devfreq->previous_freq = rknpu_dev->current_freq;
if (rknpu_dev->devfreq->suspend_freq)
rknpu_dev->devfreq->resume_freq = rknpu_dev->current_freq;
rknpu_dev->devfreq->last_status.current_frequency =
rknpu_dev->current_freq;
rknpu_dev->devfreq->last_status.total_time = 1;
rknpu_dev->devfreq->last_status.busy_time = 1;
return 0;
err_remove_governor:
devfreq_remove_governor(&devfreq_rknpu_ondemand);
err_uinit_table:
#if KERNEL_VERSION(5, 10, 198) <= LINUX_VERSION_CODE
rockchip_uninit_opp_table(dev, info);
#else
dev_pm_opp_of_remove_table(dev);
#endif
return ret;
}
EXPORT_SYMBOL(rknpu_devfreq_init);
int rknpu_devfreq_runtime_suspend(struct device *dev)
{
struct rknpu_device *rknpu_dev = dev_get_drvdata(dev);
struct rockchip_opp_info *opp_info = &rknpu_dev->opp_info;
if (opp_info->is_scmi_clk) {
if (clk_set_rate(opp_info->clk, POWER_DOWN_FREQ))
LOG_DEV_ERROR(dev, "failed to restore clk rate\n");
}
opp_info->current_rm = UINT_MAX;
return 0;
}
EXPORT_SYMBOL(rknpu_devfreq_runtime_suspend);
int rknpu_devfreq_runtime_resume(struct device *dev)
{
struct rknpu_device *rknpu_dev = dev_get_drvdata(dev);
struct rockchip_opp_info *opp_info = &rknpu_dev->opp_info;
int ret = 0;
if (!rknpu_dev->current_freq || !rknpu_dev->current_volt)
return 0;
ret = clk_bulk_prepare_enable(opp_info->nclocks, opp_info->clocks);
if (ret) {
LOG_DEV_INFO(dev, "failed to enable opp clks\n");
return ret;
}
if (opp_info->data && opp_info->data->set_read_margin)
opp_info->data->set_read_margin(dev, opp_info,
opp_info->target_rm);
if (opp_info->is_scmi_clk) {
if (clk_set_rate(opp_info->clk, rknpu_dev->current_freq))
LOG_DEV_ERROR(dev, "failed to set power down rate\n");
}
clk_bulk_disable_unprepare(opp_info->nclocks, opp_info->clocks);
return ret;
}
EXPORT_SYMBOL(rknpu_devfreq_runtime_resume);
#else
void rknpu_devfreq_lock(struct rknpu_device *rknpu_dev)
{
rockchip_monitor_volt_adjust_lock(rknpu_dev->mdev_info);
}
EXPORT_SYMBOL(rknpu_devfreq_lock);
void rknpu_devfreq_unlock(struct rknpu_device *rknpu_dev)
{
rockchip_monitor_volt_adjust_unlock(rknpu_dev->mdev_info);
}
EXPORT_SYMBOL(rknpu_devfreq_unlock);
static int npu_opp_helper(struct dev_pm_set_opp_data *data)
{
struct device *dev = data->dev;
struct dev_pm_opp_supply *old_supply_vdd = &data->old_opp.supplies[0];
struct dev_pm_opp_supply *old_supply_mem = &data->old_opp.supplies[1];
struct dev_pm_opp_supply *new_supply_vdd = &data->new_opp.supplies[0];
struct dev_pm_opp_supply *new_supply_mem = &data->new_opp.supplies[1];
struct regulator *vdd_reg = data->regulators[0];
struct regulator *mem_reg = data->regulators[1];
struct clk *clk = data->clk;
struct rknpu_device *rknpu_dev = dev_get_drvdata(dev);
struct rockchip_opp_info *opp_info = &rknpu_dev->opp_info;
unsigned long old_freq = data->old_opp.rate;
unsigned long new_freq = data->new_opp.rate;
bool is_set_rm = true;
bool is_set_clk = true;
u32 target_rm = UINT_MAX;
int ret = 0;
if (!pm_runtime_active(dev)) {
is_set_rm = false;
if (opp_info->scmi_clk)
is_set_clk = false;
}
ret = clk_bulk_prepare_enable(opp_info->num_clks, opp_info->clks);
if (ret < 0) {
LOG_DEV_ERROR(dev, "failed to enable opp clks\n");
return ret;
}
rockchip_get_read_margin(dev, opp_info, new_supply_vdd->u_volt,
&target_rm);
/* Change frequency */
LOG_DEV_DEBUG(dev, "switching OPP: %lu Hz --> %lu Hz\n", old_freq,
new_freq);
/* Scaling up? Scale voltage before frequency */
if (new_freq >= old_freq) {
rockchip_set_intermediate_rate(dev, opp_info, clk, old_freq,
new_freq, true, is_set_clk);
ret = regulator_set_voltage(mem_reg, new_supply_mem->u_volt,
INT_MAX);
if (ret) {
LOG_DEV_ERROR(dev,
"failed to set volt %lu uV for mem reg\n",
new_supply_mem->u_volt);
goto restore_voltage;
}
ret = regulator_set_voltage(vdd_reg, new_supply_vdd->u_volt,
INT_MAX);
if (ret) {
LOG_DEV_ERROR(dev,
"failed to set volt %lu uV for vdd reg\n",
new_supply_vdd->u_volt);
goto restore_voltage;
}
rockchip_set_read_margin(dev, opp_info, target_rm, is_set_rm);
if (is_set_clk && clk_set_rate(clk, new_freq)) {
ret = -EINVAL;
LOG_DEV_ERROR(dev, "failed to set clk rate: %d\n", ret);
goto restore_rm;
}
/* Scaling down? Scale voltage after frequency */
} else {
rockchip_set_intermediate_rate(dev, opp_info, clk, old_freq,
new_freq, false, is_set_clk);
rockchip_set_read_margin(dev, opp_info, target_rm, is_set_rm);
if (is_set_clk && clk_set_rate(clk, new_freq)) {
ret = -EINVAL;
LOG_DEV_ERROR(dev, "failed to set clk rate: %d\n", ret);
goto restore_rm;
}
ret = regulator_set_voltage(vdd_reg, new_supply_vdd->u_volt,
INT_MAX);
if (ret) {
LOG_DEV_ERROR(dev,
"failed to set volt %lu uV for vdd reg\n",
new_supply_vdd->u_volt);
goto restore_freq;
}
ret = regulator_set_voltage(mem_reg, new_supply_mem->u_volt,
INT_MAX);
if (ret) {
LOG_DEV_ERROR(dev,
"failed to set volt %lu uV for mem reg\n",
new_supply_mem->u_volt);
goto restore_freq;
}
}
clk_bulk_disable_unprepare(opp_info->num_clks, opp_info->clks);
return 0;
restore_freq:
if (is_set_clk && clk_set_rate(clk, old_freq))
LOG_DEV_ERROR(dev, "failed to restore old-freq %lu Hz\n",
old_freq);
restore_rm:
rockchip_get_read_margin(dev, opp_info, old_supply_vdd->u_volt,
&target_rm);
rockchip_set_read_margin(dev, opp_info, opp_info->current_rm,
is_set_rm);
restore_voltage:
regulator_set_voltage(mem_reg, old_supply_mem->u_volt, INT_MAX);
regulator_set_voltage(vdd_reg, old_supply_vdd->u_volt, INT_MAX);
clk_bulk_disable_unprepare(opp_info->num_clks, opp_info->clks);
return ret;
}
static int npu_devfreq_target(struct device *dev, unsigned long *freq,
u32 flags)
{
struct rknpu_device *rknpu_dev = dev_get_drvdata(dev);
struct dev_pm_opp *opp;
unsigned long opp_volt;
int ret = 0;
if (!npu_mdevp.is_checked)
return -EINVAL;
opp = devfreq_recommended_opp(dev, freq, flags);
if (IS_ERR(opp))
return PTR_ERR(opp);
opp_volt = dev_pm_opp_get_voltage(opp);
dev_pm_opp_put(opp);
rockchip_monitor_volt_adjust_lock(rknpu_dev->mdev_info);
ret = dev_pm_opp_set_rate(dev, *freq);
if (!ret) {
rknpu_dev->current_freq = *freq;
if (rknpu_dev->devfreq)
rknpu_dev->devfreq->last_status.current_frequency =
*freq;
rknpu_dev->current_volt = opp_volt;
LOG_DEV_DEBUG(dev, "set rknpu freq: %lu, volt: %lu\n",
rknpu_dev->current_freq, rknpu_dev->current_volt);
}
rockchip_monitor_volt_adjust_unlock(rknpu_dev->mdev_info);
return ret;
}
static unsigned long npu_get_static_power(struct devfreq *devfreq,
unsigned long voltage)
{
struct device *dev = devfreq->dev.parent;
struct rknpu_device *rknpu_dev = dev_get_drvdata(dev);
if (!rknpu_dev->model_data)
return 0;
return rockchip_ipa_get_static_power(rknpu_dev->model_data, voltage);
}
static struct devfreq_cooling_power npu_cooling_power = {
.get_static_power = &npu_get_static_power,
};
int rknpu_devfreq_init(struct rknpu_device *rknpu_dev)
{
struct device *dev = rknpu_dev->dev;
struct devfreq_dev_profile *dp = &npu_devfreq_profile;
struct dev_pm_opp *opp;
struct opp_table *reg_table = NULL;
struct opp_table *opp_table = NULL;
const char *const reg_names[] = { "rknpu", "mem" };
int ret = -EINVAL;
if (strstr(__clk_get_name(rknpu_dev->clks[0].clk), "scmi"))
rknpu_dev->opp_info.scmi_clk = rknpu_dev->clks[0].clk;
if (of_find_property(dev->of_node, "rknpu-supply", NULL) &&
of_find_property(dev->of_node, "mem-supply", NULL)) {
reg_table = dev_pm_opp_set_regulators(dev, reg_names, 2);
if (IS_ERR(reg_table))
return PTR_ERR(reg_table);
opp_table =
dev_pm_opp_register_set_opp_helper(dev, npu_opp_helper);
if (IS_ERR(opp_table)) {
dev_pm_opp_put_regulators(reg_table);
return PTR_ERR(opp_table);
}
} else {
reg_table = dev_pm_opp_set_regulators(dev, reg_names, 1);
if (IS_ERR(reg_table))
return PTR_ERR(reg_table);
}
rockchip_get_opp_data(rockchip_npu_of_match, &rknpu_dev->opp_info);
ret = rockchip_init_opp_table(dev, &rknpu_dev->opp_info, "npu_leakage",
"rknpu");
if (ret) {
LOG_DEV_ERROR(dev, "failed to init_opp_table\n");
return ret;
}
rknpu_dev->current_freq = clk_get_rate(rknpu_dev->clks[0].clk);
opp = devfreq_recommended_opp(dev, &rknpu_dev->current_freq, 0);
if (IS_ERR(opp)) {
ret = PTR_ERR(opp);
goto err_remove_table;
}
dev_pm_opp_put(opp);
dp->initial_freq = rknpu_dev->current_freq;
ret = devfreq_add_governor(&devfreq_rknpu_ondemand);
if (ret) {
LOG_DEV_ERROR(dev, "failed to add rknpu_ondemand governor\n");
goto err_remove_table;
}
rknpu_dev->devfreq = devm_devfreq_add_device(dev, dp, "rknpu_ondemand",
(void *)rknpu_dev);
if (IS_ERR(rknpu_dev->devfreq)) {
LOG_DEV_ERROR(dev, "failed to add devfreq\n");
ret = PTR_ERR(rknpu_dev->devfreq);
goto err_remove_governor;
}
npu_mdevp.data = rknpu_dev->devfreq;
npu_mdevp.opp_info = &rknpu_dev->opp_info;
rknpu_dev->mdev_info =
rockchip_system_monitor_register(dev, &npu_mdevp);
if (IS_ERR(rknpu_dev->mdev_info)) {
LOG_DEV_DEBUG(dev, "without system monitor\n");
rknpu_dev->mdev_info = NULL;
npu_mdevp.is_checked = true;
}
rknpu_dev->current_freq = clk_get_rate(rknpu_dev->clks[0].clk);
rknpu_dev->ondemand_freq = rknpu_dev->current_freq;
rknpu_dev->current_volt = regulator_get_voltage(rknpu_dev->vdd);
rknpu_dev->devfreq->previous_freq = rknpu_dev->current_freq;
if (rknpu_dev->devfreq->suspend_freq)
rknpu_dev->devfreq->resume_freq = rknpu_dev->current_freq;
rknpu_dev->devfreq->last_status.current_frequency =
rknpu_dev->current_freq;
rknpu_dev->devfreq->last_status.total_time = 1;
rknpu_dev->devfreq->last_status.busy_time = 1;
of_property_read_u32(dev->of_node, "dynamic-power-coefficient",
(u32 *)&npu_cooling_power.dyn_power_coeff);
rknpu_dev->model_data =
rockchip_ipa_power_model_init(dev, "npu_leakage");
if (IS_ERR_OR_NULL(rknpu_dev->model_data)) {
rknpu_dev->model_data = NULL;
LOG_DEV_ERROR(dev, "failed to initialize power model\n");
} else if (rknpu_dev->model_data->dynamic_coefficient) {
npu_cooling_power.dyn_power_coeff =
rknpu_dev->model_data->dynamic_coefficient;
}
if (!npu_cooling_power.dyn_power_coeff) {
LOG_DEV_ERROR(dev, "failed to get dynamic-coefficient\n");
goto out;
}
rknpu_dev->devfreq_cooling = of_devfreq_cooling_register_power(
dev->of_node, rknpu_dev->devfreq, &npu_cooling_power);
if (IS_ERR_OR_NULL(rknpu_dev->devfreq_cooling))
LOG_DEV_ERROR(dev, "failed to register cooling device\n");
out:
return 0;
err_remove_governor:
devfreq_remove_governor(&devfreq_rknpu_ondemand);
err_remove_table:
#if KERNEL_VERSION(5, 10, 198) <= LINUX_VERSION_CODE
rockchip_uninit_opp_table(dev, &rknpu_dev->opp_info);
#else
dev_pm_opp_of_remove_table(dev);
#endif
rknpu_dev->devfreq = NULL;
return ret;
}
EXPORT_SYMBOL(rknpu_devfreq_init);
int rknpu_devfreq_runtime_suspend(struct device *dev)
{
struct rknpu_device *rknpu_dev = dev_get_drvdata(dev);
struct rockchip_opp_info *opp_info = &rknpu_dev->opp_info;
if (opp_info->scmi_clk) {
if (clk_set_rate(opp_info->scmi_clk, POWER_DOWN_FREQ))
LOG_DEV_ERROR(dev, "failed to restore clk rate\n");
}
opp_info->current_rm = UINT_MAX;
return 0;
}
EXPORT_SYMBOL(rknpu_devfreq_runtime_suspend);
int rknpu_devfreq_runtime_resume(struct device *dev)
{
struct rknpu_device *rknpu_dev = dev_get_drvdata(dev);
struct rockchip_opp_info *opp_info = &rknpu_dev->opp_info;
int ret = 0;
if (!rknpu_dev->current_freq || !rknpu_dev->current_volt)
return 0;
ret = clk_bulk_prepare_enable(opp_info->num_clks, opp_info->clks);
if (ret) {
LOG_DEV_INFO(dev, "failed to enable opp clks\n");
return ret;
}
if (opp_info->data && opp_info->data->set_read_margin)
opp_info->data->set_read_margin(dev, opp_info,
opp_info->target_rm);
if (opp_info->scmi_clk) {
if (clk_set_rate(opp_info->scmi_clk, rknpu_dev->current_freq))
LOG_DEV_ERROR(dev, "failed to set power down rate\n");
}
clk_bulk_disable_unprepare(opp_info->num_clks, opp_info->clks);
return ret;
}
EXPORT_SYMBOL(rknpu_devfreq_runtime_resume);
#endif
void rknpu_devfreq_remove(struct rknpu_device *rknpu_dev)
{
if (rknpu_dev->mdev_info) {
rockchip_system_monitor_unregister(rknpu_dev->mdev_info);
rknpu_dev->mdev_info = NULL;
}
if (rknpu_dev->devfreq)
devfreq_remove_governor(&devfreq_rknpu_ondemand);
#if KERNEL_VERSION(5, 10, 198) <= LINUX_VERSION_CODE
rockchip_uninit_opp_table(rknpu_dev->dev, &rknpu_dev->opp_info);
#else
dev_pm_opp_of_remove_table(rknpu_dev->dev);
#endif
}
EXPORT_SYMBOL(rknpu_devfreq_remove);

1302
target/linux/rockchip/rk35xx/files/drivers/rknpu/rknpu_drv.c
View File

File diff suppressed because it is too large Load Diff

787
target/linux/rockchip/rk35xx/files/drivers/rknpu/rknpu_gem.c
View File

File diff suppressed because it is too large Load Diff

619
target/linux/rockchip/rk35xx/files/drivers/rknpu/rknpu_iommu.c Normal file
View File

@ -0,0 +1,619 @@
// SPDX-License-Identifier: GPL-2.0
/*
* Copyright (C) Rockchip Electronics Co.Ltd
* Author: Felix Zeng <felix.zeng@rock-chips.com>
*/
#include <linux/dma-map-ops.h>
#include <linux/delay.h>
#include <linux/jiffies.h>
#include "rknpu_iommu.h"
#define RKNPU_SWITCH_DOMAIN_WAIT_TIME_MS 6000
dma_addr_t rknpu_iommu_dma_alloc_iova(struct iommu_domain *domain, size_t size,
u64 dma_limit, struct device *dev,
bool size_aligned)
{
struct rknpu_iommu_dma_cookie *cookie = (void *)domain->iova_cookie;
struct iova_domain *iovad = &cookie->iovad;
unsigned long shift, iova_len, iova = 0;
unsigned long limit_pfn;
struct iova *new_iova = NULL;
bool alloc_fast = size_aligned;
shift = iova_shift(iovad);
iova_len = size >> shift;
#if KERNEL_VERSION(6, 1, 0) > LINUX_VERSION_CODE
/*
* Freeing non-power-of-two-sized allocations back into the IOVA caches
* will come back to bite us badly, so we have to waste a bit of space
* rounding up anything cacheable to make sure that can't happen. The
* order of the unadjusted size will still match upon freeing.
*/
if (iova_len < (1 << (IOVA_RANGE_CACHE_MAX_SIZE - 1)))
iova_len = roundup_pow_of_two(iova_len);
#endif
#if (KERNEL_VERSION(5, 10, 0) <= LINUX_VERSION_CODE)
dma_limit = min_not_zero(dma_limit, dev->bus_dma_limit);
#else
if (dev->bus_dma_mask)
dma_limit &= dev->bus_dma_mask;
#endif
if (domain->geometry.force_aperture)
dma_limit =
min_t(u64, dma_limit, domain->geometry.aperture_end);
#if (KERNEL_VERSION(5, 4, 0) <= LINUX_VERSION_CODE)
limit_pfn = dma_limit >> shift;
#else
limit_pfn = min_t(dma_addr_t, dma_limit >> shift, iovad->end_pfn);
#endif
if (alloc_fast) {
iova = alloc_iova_fast(iovad, iova_len, limit_pfn, true);
} else {
new_iova = alloc_iova(iovad, iova_len, limit_pfn, size_aligned);
if (!new_iova)
return 0;
iova = new_iova->pfn_lo;
}
return (dma_addr_t)iova << shift;
}
void rknpu_iommu_dma_free_iova(struct rknpu_iommu_dma_cookie *cookie,
dma_addr_t iova, size_t size, bool size_aligned)
{
struct iova_domain *iovad = &cookie->iovad;
bool alloc_fast = size_aligned;
if (alloc_fast)
free_iova_fast(iovad, iova_pfn(iovad, iova),
size >> iova_shift(iovad));
else
free_iova(iovad, iova_pfn(iovad, iova));
}
static int rknpu_dma_info_to_prot(enum dma_data_direction dir, bool coherent)
{
int prot = coherent ? IOMMU_CACHE : 0;
switch (dir) {
case DMA_BIDIRECTIONAL:
return prot | IOMMU_READ | IOMMU_WRITE;
case DMA_TO_DEVICE:
return prot | IOMMU_READ;
case DMA_FROM_DEVICE:
return prot | IOMMU_WRITE;
default:
return 0;
}
}
/*
* Prepare a successfully-mapped scatterlist to give back to the caller.
*
* At this point the segments are already laid out by iommu_dma_map_sg() to
* avoid individually crossing any boundaries, so we merely need to check a
* segment's start address to avoid concatenating across one.
*/
static int __finalise_sg(struct device *dev, struct scatterlist *sg, int nents,
dma_addr_t dma_addr)
{
struct scatterlist *s, *cur = sg;
unsigned long seg_mask = dma_get_seg_boundary(dev);
unsigned int cur_len = 0, max_len = dma_get_max_seg_size(dev);
int i, count = 0;
for_each_sg(sg, s, nents, i) {
/* Restore this segment's original unaligned fields first */
#if KERNEL_VERSION(6, 1, 0) <= LINUX_VERSION_CODE
dma_addr_t s_dma_addr = sg_dma_address(s);
#endif
unsigned int s_iova_off = sg_dma_address(s);
unsigned int s_length = sg_dma_len(s);
unsigned int s_iova_len = s->length;
sg_dma_address(s) = DMA_MAPPING_ERROR;
sg_dma_len(s) = 0;
#if KERNEL_VERSION(6, 1, 0) <= LINUX_VERSION_CODE
if (sg_is_dma_bus_address(s)) {
if (i > 0)
cur = sg_next(cur);
sg_dma_unmark_bus_address(s);
sg_dma_address(cur) = s_dma_addr;
sg_dma_len(cur) = s_length;
sg_dma_mark_bus_address(cur);
count++;
cur_len = 0;
continue;
}
#endif
s->offset += s_iova_off;
s->length = s_length;
/*
* Now fill in the real DMA data. If...
* - there is a valid output segment to append to
* - and this segment starts on an IOVA page boundary
* - but doesn't fall at a segment boundary
* - and wouldn't make the resulting output segment too long
*/
if (cur_len && !s_iova_off && (dma_addr & seg_mask) &&
(max_len - cur_len >= s_length)) {
/* ...then concatenate it with the previous one */
cur_len += s_length;
} else {
/* Otherwise start the next output segment */
if (i > 0)
cur = sg_next(cur);
cur_len = s_length;
count++;
sg_dma_address(cur) = dma_addr + s_iova_off;
}
sg_dma_len(cur) = cur_len;
dma_addr += s_iova_len;
if (s_length + s_iova_off < s_iova_len)
cur_len = 0;
}
return count;
}
/*
* If mapping failed, then just restore the original list,
* but making sure the DMA fields are invalidated.
*/
static void __invalidate_sg(struct scatterlist *sg, int nents)
{
struct scatterlist *s;
int i;
#if KERNEL_VERSION(6, 1, 0) <= LINUX_VERSION_CODE
for_each_sg(sg, s, nents, i) {
if (sg_is_dma_bus_address(s)) {
sg_dma_unmark_bus_address(s);
} else {
if (sg_dma_address(s) != DMA_MAPPING_ERROR)
s->offset += sg_dma_address(s);
if (sg_dma_len(s))
s->length = sg_dma_len(s);
}
sg_dma_address(s) = DMA_MAPPING_ERROR;
sg_dma_len(s) = 0;
}
#else
for_each_sg(sg, s, nents, i) {
if (sg_dma_address(s) != DMA_MAPPING_ERROR)
s->offset += sg_dma_address(s);
if (sg_dma_len(s))
s->length = sg_dma_len(s);
sg_dma_address(s) = DMA_MAPPING_ERROR;
sg_dma_len(s) = 0;
}
#endif
}
int rknpu_iommu_dma_map_sg(struct device *dev, struct scatterlist *sg,
int nents, enum dma_data_direction dir,
bool iova_aligned)
{
struct iommu_domain *domain = iommu_get_domain_for_dev(dev);
struct rknpu_iommu_dma_cookie *cookie = (void *)domain->iova_cookie;
struct iova_domain *iovad = &cookie->iovad;
struct scatterlist *s = NULL, *prev = NULL;
int prot = rknpu_dma_info_to_prot(dir, dev_is_dma_coherent(dev));
dma_addr_t iova;
unsigned long iova_len = 0;
unsigned long mask = dma_get_seg_boundary(dev);
ssize_t ret = -EINVAL;
int i = 0;
if (iova_aligned)
return dma_map_sg(dev, sg, nents, dir);
/*
* Work out how much IOVA space we need, and align the segments to
* IOVA granules for the IOMMU driver to handle. With some clever
* trickery we can modify the list in-place, but reversibly, by
* stashing the unaligned parts in the as-yet-unused DMA fields.
*/
for_each_sg(sg, s, nents, i) {
size_t s_iova_off = iova_offset(iovad, s->offset);
size_t s_length = s->length;
size_t pad_len = (mask - iova_len + 1) & mask;
sg_dma_address(s) = s_iova_off;
sg_dma_len(s) = s_length;
s->offset -= s_iova_off;
s_length = iova_align(iovad, s_length + s_iova_off);
s->length = s_length;
/*
* Due to the alignment of our single IOVA allocation, we can
* depend on these assumptions about the segment boundary mask:
* - If mask size >= IOVA size, then the IOVA range cannot
* possibly fall across a boundary, so we don't care.
* - If mask size < IOVA size, then the IOVA range must start
* exactly on a boundary, therefore we can lay things out
* based purely on segment lengths without needing to know
* the actual addresses beforehand.
* - The mask must be a power of 2, so pad_len == 0 if
* iova_len == 0, thus we cannot dereference prev the first
* time through here (i.e. before it has a meaningful value).
*/
if (pad_len && pad_len < s_length - 1) {
prev->length += pad_len;
iova_len += pad_len;
}
iova_len += s_length;
prev = s;
}
if (!iova_len) {
ret = __finalise_sg(dev, sg, nents, 0);
goto out;
}
iova = rknpu_iommu_dma_alloc_iova(domain, iova_len, dma_get_mask(dev),
dev, iova_aligned);
if (!iova) {
ret = -ENOMEM;
LOG_ERROR("failed to allocate IOVA: %zd\n", ret);
goto out_restore_sg;
}
ret = iommu_map_sg(domain, iova, sg, nents, prot);
if (ret < 0 || ret < iova_len) {
LOG_ERROR("failed to map SG: %zd\n", ret);
goto out_free_iova;
}
return __finalise_sg(dev, sg, nents, iova);
out_free_iova:
rknpu_iommu_dma_free_iova(cookie, iova, iova_len, iova_aligned);
out_restore_sg:
__invalidate_sg(sg, nents);
out:
if (ret < 0)
ret = 0;
return ret;
}
void rknpu_iommu_dma_unmap_sg(struct device *dev, struct scatterlist *sg,
int nents, enum dma_data_direction dir,
bool iova_aligned)
{
struct iommu_domain *domain = iommu_get_domain_for_dev(dev);
struct rknpu_iommu_dma_cookie *cookie = (void *)domain->iova_cookie;
struct iova_domain *iovad = &cookie->iovad;
size_t iova_off = 0;
dma_addr_t end = 0, start = 0;
struct scatterlist *tmp = NULL;
dma_addr_t dma_addr = 0;
size_t size = 0;
int i = 0;
if (iova_aligned)
return dma_unmap_sg(dev, sg, nents, dir);
#if KERNEL_VERSION(6, 1, 0) <= LINUX_VERSION_CODE
/*
* The scatterlist segments are mapped into a single
* contiguous IOVA allocation, the start and end points
* just have to be determined.
*/
for_each_sg(sg, tmp, nents, i) {
if (sg_is_dma_bus_address(tmp)) {
sg_dma_unmark_bus_address(tmp);
continue;
}
if (sg_dma_len(tmp) == 0)
break;
start = sg_dma_address(tmp);
break;
}
nents -= i;
for_each_sg(tmp, tmp, nents, i) {
if (sg_is_dma_bus_address(tmp)) {
sg_dma_unmark_bus_address(tmp);
continue;
}
if (sg_dma_len(tmp) == 0)
break;
end = sg_dma_address(tmp) + sg_dma_len(tmp);
}
#else
start = sg_dma_address(sg);
for_each_sg(sg_next(sg), tmp, nents - 1, i) {
if (sg_dma_len(tmp) == 0)
break;
sg = tmp;
}
end = sg_dma_address(sg) + sg_dma_len(sg);
#endif
dma_addr = start;
size = end - start;
iova_off = iova_offset(iovad, start);
if (end) {
dma_addr -= iova_off;
size = iova_align(iovad, size + iova_off);
iommu_unmap(domain, dma_addr, size);
rknpu_iommu_dma_free_iova(cookie, dma_addr, size, iova_aligned);
}
}
#if defined(CONFIG_IOMMU_API) && defined(CONFIG_NO_GKI)
#if KERNEL_VERSION(6, 1, 0) <= LINUX_VERSION_CODE
struct iommu_group {
struct kobject kobj;
struct kobject *devices_kobj;
struct list_head devices;
#ifdef __ANDROID_COMMON_KERNEL__
struct xarray pasid_array;
#endif
struct mutex mutex;
void *iommu_data;
void (*iommu_data_release)(void *iommu_data);
char *name;
int id;
struct iommu_domain *default_domain;
struct iommu_domain *blocking_domain;
struct iommu_domain *domain;
struct list_head entry;
unsigned int owner_cnt;
void *owner;
};
#else
struct iommu_group {
struct kobject kobj;
struct kobject *devices_kobj;
struct list_head devices;
struct mutex mutex;
struct blocking_notifier_head notifier;
void *iommu_data;
void (*iommu_data_release)(void *iommu_data);
char *name;
int id;
struct iommu_domain *default_domain;
struct iommu_domain *domain;
struct list_head entry;
};
#endif
int rknpu_iommu_init_domain(struct rknpu_device *rknpu_dev)
{
// init domain 0
if (!rknpu_dev->iommu_domains[0]) {
rknpu_dev->iommu_domain_id = 0;
rknpu_dev->iommu_domains[rknpu_dev->iommu_domain_id] =
iommu_get_domain_for_dev(rknpu_dev->dev);
rknpu_dev->iommu_domain_num = 1;
}
return 0;
}
int rknpu_iommu_switch_domain(struct rknpu_device *rknpu_dev, int domain_id)
{
struct iommu_domain *src_domain = NULL;
struct iommu_domain *dst_domain = NULL;
struct bus_type *bus = NULL;
int src_domain_id = 0;
int ret = -EINVAL;
if (!rknpu_dev->iommu_en)
return -EINVAL;
if (domain_id < 0 || domain_id > (RKNPU_MAX_IOMMU_DOMAIN_NUM - 1)) {
LOG_DEV_ERROR(
rknpu_dev->dev,
"invalid iommu domain id: %d, reuse domain id: %d\n",
domain_id, rknpu_dev->iommu_domain_id);
return -EINVAL;
}
bus = rknpu_dev->dev->bus;
if (!bus)
return -EFAULT;
src_domain_id = rknpu_dev->iommu_domain_id;
if (domain_id == src_domain_id) {
return 0;
}
src_domain = iommu_get_domain_for_dev(rknpu_dev->dev);
if (src_domain != rknpu_dev->iommu_domains[src_domain_id]) {
LOG_DEV_ERROR(
rknpu_dev->dev,
"mismatch domain get from iommu_get_domain_for_dev\n");
return -EINVAL;
}
dst_domain = rknpu_dev->iommu_domains[domain_id];
if (dst_domain != NULL) {
iommu_detach_device(src_domain, rknpu_dev->dev);
ret = iommu_attach_device(dst_domain, rknpu_dev->dev);
if (ret) {
LOG_DEV_ERROR(
rknpu_dev->dev,
"failed to attach dst iommu domain, id: %d, ret: %d\n",
domain_id, ret);
if (iommu_attach_device(src_domain, rknpu_dev->dev)) {
LOG_DEV_ERROR(
rknpu_dev->dev,
"failed to reattach src iommu domain, id: %d\n",
src_domain_id);
}
return ret;
}
rknpu_dev->iommu_domain_id = domain_id;
} else {
uint64_t dma_limit = 1ULL << 32;
dst_domain = iommu_domain_alloc(bus);
if (!dst_domain) {
LOG_DEV_ERROR(rknpu_dev->dev,
"failed to allocate iommu domain\n");
return -EIO;
}
// init domain iova_cookie
iommu_get_dma_cookie(dst_domain);
iommu_detach_device(src_domain, rknpu_dev->dev);
ret = iommu_attach_device(dst_domain, rknpu_dev->dev);
if (ret) {
LOG_DEV_ERROR(
rknpu_dev->dev,
"failed to attach iommu domain, id: %d, ret: %d\n",
domain_id, ret);
iommu_domain_free(dst_domain);
return ret;
}
// set domain type to dma domain
dst_domain->type |= __IOMMU_DOMAIN_DMA_API;
// iommu dma init domain
iommu_setup_dma_ops(rknpu_dev->dev, 0, dma_limit);
rknpu_dev->iommu_domain_id = domain_id;
rknpu_dev->iommu_domains[domain_id] = dst_domain;
rknpu_dev->iommu_domain_num++;
}
// reset default iommu domain
rknpu_dev->iommu_group->default_domain = dst_domain;
LOG_INFO("switch iommu domain from %d to %d\n", src_domain_id,
domain_id);
return ret;
}
int rknpu_iommu_domain_get_and_switch(struct rknpu_device *rknpu_dev,
int domain_id)
{
unsigned long timeout_jiffies =
msecs_to_jiffies(RKNPU_SWITCH_DOMAIN_WAIT_TIME_MS);
unsigned long start = jiffies;
int ret = -EINVAL;
while (true) {
mutex_lock(&rknpu_dev->domain_lock);
if (domain_id == rknpu_dev->iommu_domain_id) {
atomic_inc(&rknpu_dev->iommu_domain_refcount);
mutex_unlock(&rknpu_dev->domain_lock);
break;
}
if (atomic_read(&rknpu_dev->iommu_domain_refcount) == 0) {
ret = rknpu_iommu_switch_domain(rknpu_dev, domain_id);
if (ret) {
LOG_DEV_ERROR(
rknpu_dev->dev,
"failed to switch iommu domain, id: %d, ret: %d\n",
domain_id, ret);
mutex_unlock(&rknpu_dev->domain_lock);
return ret;
}
atomic_inc(&rknpu_dev->iommu_domain_refcount);
mutex_unlock(&rknpu_dev->domain_lock);
break;
}
mutex_unlock(&rknpu_dev->domain_lock);
usleep_range(10, 100);
if (time_after(jiffies, start + timeout_jiffies)) {
LOG_DEV_ERROR(
rknpu_dev->dev,
"switch iommu domain time out, failed to switch iommu domain, id: %d\n",
domain_id);
return -EINVAL;
}
}
return 0;
}
int rknpu_iommu_domain_put(struct rknpu_device *rknpu_dev)
{
atomic_dec(&rknpu_dev->iommu_domain_refcount);
return 0;
}
void rknpu_iommu_free_domains(struct rknpu_device *rknpu_dev)
{
int i = 0;
if (rknpu_iommu_domain_get_and_switch(rknpu_dev, 0)) {
LOG_DEV_ERROR(rknpu_dev->dev, "%s error\n", __func__);
return;
}
for (i = 1; i < RKNPU_MAX_IOMMU_DOMAIN_NUM; i++) {
struct iommu_domain *domain = rknpu_dev->iommu_domains[i];
if (domain == NULL)
continue;
iommu_detach_device(domain, rknpu_dev->dev);
iommu_domain_free(domain);
rknpu_dev->iommu_domains[i] = NULL;
}
rknpu_iommu_domain_put(rknpu_dev);
}
#else
int rknpu_iommu_init_domain(struct rknpu_device *rknpu_dev)
{
return 0;
}
int rknpu_iommu_switch_domain(struct rknpu_device *rknpu_dev, int domain_id)
{
return 0;
}
int rknpu_iommu_domain_get_and_switch(struct rknpu_device *rknpu_dev,
int domain_id)
{
return 0;
}
int rknpu_iommu_domain_put(struct rknpu_device *rknpu_dev)
{
return 0;
}
void rknpu_iommu_free_domains(struct rknpu_device *rknpu_dev)
{
}
#endif

439
target/linux/rockchip/rk35xx/files/drivers/rknpu/rknpu_job.c
View File

@ -14,8 +14,9 @@
#include "rknpu_reset.h"
#include "rknpu_gem.h"
#include "rknpu_fence.h"
#include "rknpu_job.h"
#include "rknpu_mem.h"
#include "rknpu_iommu.h"
#include "rknpu_job.h"
#define _REG_READ(base, offset) readl(base + (offset))
#define _REG_WRITE(base, value, offset) writel(value, base + (offset))
@ -23,16 +24,25 @@
#define REG_READ(offset) _REG_READ(rknpu_core_base, offset)
#define REG_WRITE(value, offset) _REG_WRITE(rknpu_core_base, value, offset)
static int rknpu_core_index(int core_mask)
static int rknpu_wait_core_index(int core_mask)
{
int index = 0;
if (core_mask & RKNPU_CORE0_MASK)
switch (core_mask) {
case RKNPU_CORE0_MASK:
case RKNPU_CORE0_MASK | RKNPU_CORE1_MASK:
case RKNPU_CORE0_MASK | RKNPU_CORE1_MASK | RKNPU_CORE2_MASK:
index = 0;
else if (core_mask & RKNPU_CORE1_MASK)
break;
case RKNPU_CORE1_MASK:
index = 1;
else if (core_mask & RKNPU_CORE2_MASK)
break;
case RKNPU_CORE2_MASK:
index = 2;
break;
default:
break;
}
return index;
}
@ -58,14 +68,24 @@ static int rknpu_core_mask(int core_index)
return core_mask;
}
static int rknn_get_task_number(struct rknpu_job *job, int core_index)
static int rknpu_get_task_number(struct rknpu_job *job, int core_index)
{
struct rknpu_device *rknpu_dev = job->rknpu_dev;
int task_num = job->args->task_number;
if (job->use_core_num == 2)
task_num = job->args->subcore_task[core_index].task_number;
else if (job->use_core_num == 3)
task_num = job->args->subcore_task[core_index + 2].task_number;
if (core_index >= RKNPU_MAX_CORES || core_index < 0) {
LOG_ERROR("invalid rknpu core index: %d", core_index);
return 0;
}
if (rknpu_dev->config->num_irqs > 1) {
if (job->use_core_num == 1 || job->use_core_num == 2)
task_num =
job->args->subcore_task[core_index].task_number;
else if (job->use_core_num == 3)
task_num = job->args->subcore_task[core_index + 2]
.task_number;
}
return task_num;
}
@ -109,11 +129,10 @@ static inline struct rknpu_job *rknpu_job_alloc(struct rknpu_device *rknpu_dev,
struct rknpu_submit *args)
{
struct rknpu_job *job = NULL;
int i = 0;
#ifdef CONFIG_ROCKCHIP_RKNPU_DRM_GEM
struct rknpu_gem_object *task_obj = NULL;
#endif
if (rknpu_dev->config->num_irqs == 1)
args->core_mask = RKNPU_CORE0_MASK;
job = kzalloc(sizeof(*job), GFP_KERNEL);
if (!job)
@ -124,8 +143,11 @@ static inline struct rknpu_job *rknpu_job_alloc(struct rknpu_device *rknpu_dev,
job->use_core_num = (args->core_mask & RKNPU_CORE0_MASK) +
((args->core_mask & RKNPU_CORE1_MASK) >> 1) +
((args->core_mask & RKNPU_CORE2_MASK) >> 2);
job->run_count = job->use_core_num;
job->interrupt_count = job->use_core_num;
atomic_set(&job->run_count, job->use_core_num);
atomic_set(&job->interrupt_count, job->use_core_num);
job->iommu_domain_id = args->iommu_domain_id;
for (i = 0; i < rknpu_dev->config->num_irqs; i++)
atomic_set(&job->submit_count[i], 0);
#ifdef CONFIG_ROCKCHIP_RKNPU_DRM_GEM
task_obj = (struct rknpu_gem_object *)(uintptr_t)args->task_obj_addr;
if (task_obj)
@ -157,10 +179,12 @@ static inline int rknpu_job_wait(struct rknpu_job *job)
struct rknpu_submit *args = job->args;
struct rknpu_task *last_task = NULL;
struct rknpu_subcore_data *subcore_data = NULL;
struct rknpu_job *entry, *q;
void __iomem *rknpu_core_base = NULL;
int core_index = rknpu_core_index(job->args->core_mask);
int core_index = rknpu_wait_core_index(job->args->core_mask);
unsigned long flags;
int wait_count = 0;
bool continue_wait = false;
int ret = -EINVAL;
int i = 0;
@ -171,32 +195,50 @@ static inline int rknpu_job_wait(struct rknpu_job *job)
job->flags & RKNPU_JOB_DONE ||
rknpu_dev->soft_reseting,
msecs_to_jiffies(args->timeout));
if (++wait_count >= 3)
break;
} while (ret == 0 && job->in_queue[core_index]);
if (job->in_queue[core_index]) {
spin_lock_irqsave(&rknpu_dev->lock, flags);
subcore_data->task_num -= rknn_get_task_number(job, core_index);
if (job->use_core_num == 1) {
list_del_init(&job->head[core_index]);
job->in_queue[core_index] = false;
} else if (job->use_core_num > 1) {
for (i = 0; i < job->use_core_num; i++) {
if (job->in_queue[i]) {
list_del_init(&job->head[i]);
job->in_queue[i] = false;
if (ret == 0) {
int64_t elapse_time_us = 0;
spin_lock_irqsave(&rknpu_dev->irq_lock, flags);
elapse_time_us = ktime_us_delta(ktime_get(),
job->hw_commit_time);
continue_wait =
job->hw_commit_time == 0 ?
true :
(elapse_time_us < args->timeout * 1000);
spin_unlock_irqrestore(&rknpu_dev->irq_lock, flags);
LOG_ERROR(
"job: %p, mask: %#x, job iommu domain id: %d, dev iommu domain id: %d, wait_count: %d, continue wait: %d, commit elapse time: %lldus, wait time: %lldus, timeout: %uus\n",
job, args->core_mask, job->iommu_domain_id,
rknpu_dev->iommu_domain_id, wait_count,
continue_wait,
(job->hw_commit_time == 0 ? 0 : elapse_time_us),
ktime_us_delta(ktime_get(), job->timestamp),
args->timeout * 1000);
}
} while (ret == 0 && continue_wait);
last_task = job->last_task;
if (!last_task) {
spin_lock_irqsave(&rknpu_dev->irq_lock, flags);
for (i = 0; i < job->use_core_num; i++) {
subcore_data = &rknpu_dev->subcore_datas[i];
list_for_each_entry_safe(
entry, q, &subcore_data->todo_list, head[i]) {
if (entry == job) {
list_del(&job->head[i]);
break;
}
}
}
spin_unlock_irqrestore(&rknpu_dev->lock, flags);
spin_unlock_irqrestore(&rknpu_dev->irq_lock, flags);
LOG_ERROR("job commit failed\n");
return ret < 0 ? ret : -EINVAL;
}
last_task = job->last_task;
if (!last_task)
return ret < 0 ? ret : -EINVAL;
last_task->int_status = job->int_status[core_index];
if (ret <= 0) {
@ -213,7 +255,7 @@ static inline int rknpu_job_wait(struct rknpu_job *job)
LOG_ERROR(
"failed to wait job, task counter: %d, flags: %#x, ret = %d, elapsed time: %lldus\n",
args->task_counter, args->flags, ret,
ktime_to_us(ktime_sub(ktime_get(), job->timestamp)));
ktime_us_delta(ktime_get(), job->timestamp));
return ret < 0 ? ret : -ETIMEDOUT;
}
@ -222,11 +264,13 @@ static inline int rknpu_job_wait(struct rknpu_job *job)
return -EINVAL;
args->task_counter = args->task_number;
args->hw_elapse_time = job->hw_elapse_time;
return 0;
}
static inline int rknpu_job_commit_pc(struct rknpu_job *job, int core_index)
static inline int rknpu_job_subcore_commit_pc(struct rknpu_job *job,
int core_index)
{
struct rknpu_device *rknpu_dev = job->rknpu_dev;
struct rknpu_submit *args = job->args;
@ -243,15 +287,20 @@ static inline int rknpu_job_commit_pc(struct rknpu_job *job, int core_index)
struct rknpu_task *last_task = NULL;
void __iomem *rknpu_core_base = rknpu_dev->base[core_index];
int task_start = args->task_start;
int task_end = args->task_start + args->task_number - 1;
int task_end;
int task_number = args->task_number;
int task_pp_en = args->flags & RKNPU_JOB_PINGPONG ? 1 : 0;
int pc_data_amount_scale = rknpu_dev->config->pc_data_amount_scale;
int pc_task_number_bits = rknpu_dev->config->pc_task_number_bits;
int i = 0;
int submit_index = atomic_read(&job->submit_count[core_index]);
int max_submit_number = rknpu_dev->config->max_submit_number;
unsigned long flags;
if (!task_obj)
return -EINVAL;
if (!task_obj) {
job->ret = -EINVAL;
return job->ret;
}
if (rknpu_dev->config->num_irqs > 1) {
for (i = 0; i < rknpu_dev->config->num_irqs; i++) {
@ -261,38 +310,44 @@ static inline int rknpu_job_commit_pc(struct rknpu_job *job, int core_index)
}
}
if (job->use_core_num == 1) {
switch (job->use_core_num) {
case 1:
case 2:
task_start = args->subcore_task[core_index].task_start;
task_end = args->subcore_task[core_index].task_start +
args->subcore_task[core_index].task_number -
1;
task_number =
args->subcore_task[core_index].task_number;
} else if (job->use_core_num == 2) {
task_start = args->subcore_task[core_index].task_start;
task_end = args->subcore_task[core_index].task_start +
args->subcore_task[core_index].task_number -
1;
task_number =
args->subcore_task[core_index].task_number;
} else if (job->use_core_num == 3) {
break;
case 3:
task_start =
args->subcore_task[core_index + 2].task_start;
task_end =
args->subcore_task[core_index + 2].task_start +
args->subcore_task[core_index + 2].task_number -
1;
task_number =
args->subcore_task[core_index + 2].task_number;
break;
default:
LOG_ERROR("Unknown use core num %d\n",
job->use_core_num);
break;
}
}
task_start = task_start + submit_index * max_submit_number;
task_number = task_number - submit_index * max_submit_number;
task_number = task_number > max_submit_number ? max_submit_number :
task_number;
task_end = task_start + task_number - 1;
task_base = task_obj->kv_addr;
first_task = &task_base[task_start];
last_task = &task_base[task_end];
REG_WRITE(first_task->regcmd_addr, RKNPU_OFFSET_PC_DATA_ADDR);
if (rknpu_dev->config->pc_dma_ctrl) {
spin_lock_irqsave(&rknpu_dev->irq_lock, flags);
REG_WRITE(first_task->regcmd_addr, RKNPU_OFFSET_PC_DATA_ADDR);
spin_unlock_irqrestore(&rknpu_dev->irq_lock, flags);
} else {
REG_WRITE(first_task->regcmd_addr, RKNPU_OFFSET_PC_DATA_ADDR);
}
REG_WRITE((first_task->regcfg_amount + RKNPU_PC_DATA_EXTRA_AMOUNT +
pc_data_amount_scale - 1) /
@ -319,19 +374,56 @@ static inline int rknpu_job_commit_pc(struct rknpu_job *job, int core_index)
return 0;
}
static int rknpu_job_commit(struct rknpu_job *job, int core_index)
static inline int rknpu_job_subcore_commit(struct rknpu_job *job,
int core_index)
{
struct rknpu_device *rknpu_dev = job->rknpu_dev;
struct rknpu_submit *args = job->args;
void __iomem *rknpu_core_base = rknpu_dev->base[core_index];
unsigned long flags;
// switch to slave mode
REG_WRITE(0x1, RKNPU_OFFSET_PC_DATA_ADDR);
if (rknpu_dev->config->pc_dma_ctrl) {
spin_lock_irqsave(&rknpu_dev->irq_lock, flags);
REG_WRITE(0x1, RKNPU_OFFSET_PC_DATA_ADDR);
spin_unlock_irqrestore(&rknpu_dev->irq_lock, flags);
} else {
REG_WRITE(0x1, RKNPU_OFFSET_PC_DATA_ADDR);
}
if (!(args->flags & RKNPU_JOB_PC))
return -EINVAL;
if (!(args->flags & RKNPU_JOB_PC)) {
job->ret = -EINVAL;
return job->ret;
}
return rknpu_job_commit_pc(job, core_index);
return rknpu_job_subcore_commit_pc(job, core_index);
}
static void rknpu_job_commit(struct rknpu_job *job)
{
switch (job->args->core_mask) {
case RKNPU_CORE0_MASK:
rknpu_job_subcore_commit(job, 0);
break;
case RKNPU_CORE1_MASK:
rknpu_job_subcore_commit(job, 1);
break;
case RKNPU_CORE2_MASK:
rknpu_job_subcore_commit(job, 2);
break;
case RKNPU_CORE0_MASK | RKNPU_CORE1_MASK:
rknpu_job_subcore_commit(job, 0);
rknpu_job_subcore_commit(job, 1);
break;
case RKNPU_CORE0_MASK | RKNPU_CORE1_MASK | RKNPU_CORE2_MASK:
rknpu_job_subcore_commit(job, 0);
rknpu_job_subcore_commit(job, 1);
rknpu_job_subcore_commit(job, 2);
break;
default:
LOG_ERROR("Unknown core mask: %d\n", job->args->core_mask);
break;
}
}
static void rknpu_job_next(struct rknpu_device *rknpu_dev, int core_index)
@ -356,42 +448,45 @@ static void rknpu_job_next(struct rknpu_device *rknpu_dev, int core_index)
head[core_index]);
list_del_init(&job->head[core_index]);
job->in_queue[core_index] = false;
subcore_data->job = job;
job->run_count--;
job->hw_recoder_time = ktime_get();
job->hw_commit_time = ktime_get();
job->hw_recoder_time = job->hw_commit_time;
spin_unlock_irqrestore(&rknpu_dev->irq_lock, flags);
if (job->run_count == 0) {
if (job->args->core_mask & RKNPU_CORE0_MASK)
job->ret = rknpu_job_commit(job, 0);
if (job->args->core_mask & RKNPU_CORE1_MASK)
job->ret = rknpu_job_commit(job, 1);
if (job->args->core_mask & RKNPU_CORE2_MASK)
job->ret = rknpu_job_commit(job, 2);
}
if (atomic_dec_and_test(&job->run_count))
rknpu_job_commit(job);
}
static void rknpu_job_done(struct rknpu_job *job, int ret, int core_index)
{
struct rknpu_device *rknpu_dev = job->rknpu_dev;
struct rknpu_subcore_data *subcore_data = NULL;
ktime_t now;
unsigned long flags;
ktime_t now = ktime_get();
int max_submit_number = rknpu_dev->config->max_submit_number;
if (atomic_inc_return(&job->submit_count[core_index]) <
(rknpu_get_task_number(job, core_index) + max_submit_number - 1) /
max_submit_number) {
rknpu_job_subcore_commit(job, core_index);
return;
}
subcore_data = &rknpu_dev->subcore_datas[core_index];
spin_lock_irqsave(&rknpu_dev->irq_lock, flags);
subcore_data->job = NULL;
subcore_data->task_num -= rknn_get_task_number(job, core_index);
job->interrupt_count--;
subcore_data->timer.busy_time +=
ktime_us_delta(now, job->hw_recoder_time);
subcore_data->task_num -= rknpu_get_task_number(job, core_index);
now = ktime_get();
job->hw_elapse_time = ktime_sub(now, job->hw_commit_time);
subcore_data->timer.busy_time += ktime_sub(now, job->hw_recoder_time);
spin_unlock_irqrestore(&rknpu_dev->irq_lock, flags);
if (job->interrupt_count == 0) {
if (atomic_dec_and_test(&job->interrupt_count)) {
int use_core_num = job->use_core_num;
rknpu_iommu_domain_put(rknpu_dev);
job->flags |= RKNPU_JOB_DONE;
job->ret = ret;
@ -410,47 +505,41 @@ static void rknpu_job_done(struct rknpu_job *job, int ret, int core_index)
rknpu_job_next(rknpu_dev, core_index);
}
static int rknpu_schedule_core_index(struct rknpu_device *rknpu_dev)
{
int core_num = rknpu_dev->config->num_irqs;
int task_num = rknpu_dev->subcore_datas[0].task_num;
int core_index = 0;
int i = 0;
for (i = 1; i < core_num; i++) {
if (task_num > rknpu_dev->subcore_datas[i].task_num) {
core_index = i;
task_num = rknpu_dev->subcore_datas[i].task_num;
}
}
return core_index;
}
static void rknpu_job_schedule(struct rknpu_job *job)
{
struct rknpu_device *rknpu_dev = job->rknpu_dev;
struct rknpu_subcore_data *subcore_data = NULL;
int i = 0, core_index = 0;
unsigned long flags;
int task_num_list[3] = { 0, 1, 2 };
int tmp = 0;
if ((job->args->core_mask & 0x07) == RKNPU_CORE_AUTO_MASK) {
if (rknpu_dev->subcore_datas[0].task_num >
rknpu_dev->subcore_datas[1].task_num) {
tmp = task_num_list[1];
task_num_list[1] = task_num_list[0];
task_num_list[0] = tmp;
}
if (rknpu_dev->subcore_datas[task_num_list[0]].task_num >
rknpu_dev->subcore_datas[2].task_num) {
tmp = task_num_list[2];
task_num_list[2] = task_num_list[1];
task_num_list[1] = task_num_list[0];
task_num_list[0] = tmp;
} else if (rknpu_dev->subcore_datas[task_num_list[1]].task_num >
rknpu_dev->subcore_datas[2].task_num) {
tmp = task_num_list[2];
task_num_list[2] = task_num_list[1];
task_num_list[1] = tmp;
}
if (!rknpu_dev->subcore_datas[task_num_list[0]].job)
core_index = task_num_list[0];
else if (!rknpu_dev->subcore_datas[task_num_list[1]].job)
core_index = task_num_list[1];
else if (!rknpu_dev->subcore_datas[task_num_list[2]].job)
core_index = task_num_list[2];
else
core_index = task_num_list[0];
if (job->args->core_mask == RKNPU_CORE_AUTO_MASK) {
core_index = rknpu_schedule_core_index(rknpu_dev);
job->args->core_mask = rknpu_core_mask(core_index);
job->use_core_num = 1;
job->interrupt_count = 1;
job->run_count = 1;
atomic_set(&job->run_count, job->use_core_num);
atomic_set(&job->interrupt_count, job->use_core_num);
}
if (rknpu_iommu_domain_get_and_switch(rknpu_dev, job->iommu_domain_id)) {
job->ret = -EINVAL;
return;
}
spin_lock_irqsave(&rknpu_dev->irq_lock, flags);
@ -458,8 +547,7 @@ static void rknpu_job_schedule(struct rknpu_job *job)
if (job->args->core_mask & rknpu_core_mask(i)) {
subcore_data = &rknpu_dev->subcore_datas[i];
list_add_tail(&job->head[i], &subcore_data->todo_list);
subcore_data->task_num += rknn_get_task_number(job, i);
job->in_queue[i] = true;
subcore_data->task_num += rknpu_get_task_number(job, i);
}
}
spin_unlock_irqrestore(&rknpu_dev->irq_lock, flags);
@ -474,41 +562,51 @@ static void rknpu_job_abort(struct rknpu_job *job)
{
struct rknpu_device *rknpu_dev = job->rknpu_dev;
struct rknpu_subcore_data *subcore_data = NULL;
int core_index = rknpu_core_index(job->args->core_mask);
void __iomem *rknpu_core_base = rknpu_dev->base[core_index];
unsigned long flags;
int i = 0;
rknpu_iommu_domain_put(rknpu_dev);
msleep(100);
spin_lock_irqsave(&rknpu_dev->irq_lock, flags);
for (i = 0; i < rknpu_dev->config->num_irqs; i++) {
if (job->args->core_mask & rknpu_core_mask(i)) {
subcore_data = &rknpu_dev->subcore_datas[i];
spin_lock_irqsave(&rknpu_dev->irq_lock, flags);
if (job == subcore_data->job && !job->irq_entry[i]) {
subcore_data->job = NULL;
subcore_data->task_num -=
rknn_get_task_number(job, i);
rknpu_get_task_number(job, i);
}
spin_unlock_irqrestore(&rknpu_dev->irq_lock, flags);
}
}
spin_unlock_irqrestore(&rknpu_dev->irq_lock, flags);
if (job->ret == -ETIMEDOUT) {
LOG_ERROR(
"job timeout, flags: %#x, irq status: %#x, raw status: %#x, require mask: %#x, task counter: %#x, elapsed time: %lldus\n",
job->flags, REG_READ(RKNPU_OFFSET_INT_STATUS),
REG_READ(RKNPU_OFFSET_INT_RAW_STATUS),
job->int_mask[core_index],
(REG_READ(rknpu_dev->config->pc_task_status_offset) &
rknpu_dev->config->pc_task_number_mask),
ktime_to_us(ktime_sub(ktime_get(), job->timestamp)));
LOG_ERROR("job timeout, flags: %#x:\n", job->flags);
for (i = 0; i < rknpu_dev->config->num_irqs; i++) {
if (job->args->core_mask & rknpu_core_mask(i)) {
void __iomem *rknpu_core_base =
rknpu_dev->base[i];
LOG_ERROR(
"\tcore %d irq status: %#x, raw status: %#x, require mask: %#x, task counter: %#x, elapsed time: %lldus\n",
i, REG_READ(RKNPU_OFFSET_INT_STATUS),
REG_READ(RKNPU_OFFSET_INT_RAW_STATUS),
job->int_mask[i],
(REG_READ(
rknpu_dev->config
->pc_task_status_offset) &
rknpu_dev->config->pc_task_number_mask),
ktime_us_delta(ktime_get(),
job->timestamp));
}
}
rknpu_soft_reset(rknpu_dev);
} else {
LOG_ERROR(
"job abort, flags: %#x, ret: %d, elapsed time: %lldus\n",
job->flags, job->ret,
ktime_to_us(ktime_sub(ktime_get(), job->timestamp)));
ktime_us_delta(ktime_get(), job->timestamp));
}
rknpu_job_cleanup(job);
@ -603,7 +701,6 @@ static void rknpu_job_timeout_clean(struct rknpu_device *rknpu_dev,
{
struct rknpu_job *job = NULL;
unsigned long flags;
ktime_t now = ktime_get();
struct rknpu_subcore_data *subcore_data = NULL;
int i = 0;
@ -612,7 +709,7 @@ static void rknpu_job_timeout_clean(struct rknpu_device *rknpu_dev,
subcore_data = &rknpu_dev->subcore_datas[i];
job = subcore_data->job;
if (job &&
ktime_to_ms(ktime_sub(now, job->timestamp)) >=
ktime_us_delta(ktime_get(), job->timestamp) >=
job->args->timeout) {
rknpu_soft_reset(rknpu_dev);
@ -634,7 +731,6 @@ static void rknpu_job_timeout_clean(struct rknpu_device *rknpu_dev,
struct rknpu_job,
head[i]);
list_del_init(&job->head[i]);
job->in_queue[i] = false;
} else {
job = NULL;
}
@ -658,6 +754,11 @@ static int rknpu_submit(struct rknpu_device *rknpu_dev,
return -EINVAL;
}
if (args->core_mask > rknpu_dev->config->core_mask) {
LOG_ERROR("invalid rknpu core mask: %#x", args->core_mask);
return -EINVAL;
}
job = rknpu_job_alloc(rknpu_dev, args);
if (!job) {
LOG_ERROR("failed to allocate rknpu job!\n");
@ -799,11 +900,6 @@ int rknpu_get_bw_priority(struct rknpu_device *rknpu_dev, uint32_t *priority,
{
void __iomem *base = rknpu_dev->bw_priority_base;
if (!rknpu_dev->config->bw_enable) {
LOG_WARN("Get bw_priority is not supported on this device!\n");
return 0;
}
if (!base)
return -EINVAL;
@ -828,11 +924,6 @@ int rknpu_set_bw_priority(struct rknpu_device *rknpu_dev, uint32_t priority,
{
void __iomem *base = rknpu_dev->bw_priority_base;
if (!rknpu_dev->config->bw_enable) {
LOG_WARN("Set bw_priority is not supported on this device!\n");
return 0;
}
if (!base)
return -EINVAL;
@ -855,28 +946,44 @@ int rknpu_set_bw_priority(struct rknpu_device *rknpu_dev, uint32_t priority,
int rknpu_clear_rw_amount(struct rknpu_device *rknpu_dev)
{
void __iomem *rknpu_core_base = rknpu_dev->base[0];
const struct rknpu_config *config = rknpu_dev->config;
unsigned long flags;
if (!rknpu_dev->config->bw_enable) {
if (config->amount_top == NULL) {
LOG_WARN("Clear rw_amount is not supported on this device!\n");
return 0;
}
spin_lock(&rknpu_dev->lock);
if (config->pc_dma_ctrl) {
uint32_t pc_data_addr = 0;
if (rknpu_dev->config->pc_dma_ctrl) {
uint32_t pc_data_addr = REG_READ(RKNPU_OFFSET_PC_DATA_ADDR);
spin_lock_irqsave(&rknpu_dev->irq_lock, flags);
pc_data_addr = REG_READ(RKNPU_OFFSET_PC_DATA_ADDR);
REG_WRITE(0x1, RKNPU_OFFSET_PC_DATA_ADDR);
REG_WRITE(0x80000101, RKNPU_OFFSET_CLR_ALL_RW_AMOUNT);
REG_WRITE(0x00000101, RKNPU_OFFSET_CLR_ALL_RW_AMOUNT);
REG_WRITE(0x80000101, config->amount_top->offset_clr_all);
REG_WRITE(0x00000101, config->amount_top->offset_clr_all);
if (config->amount_core) {
REG_WRITE(0x80000101,
config->amount_core->offset_clr_all);
REG_WRITE(0x00000101,
config->amount_core->offset_clr_all);
}
REG_WRITE(pc_data_addr, RKNPU_OFFSET_PC_DATA_ADDR);
spin_unlock_irqrestore(&rknpu_dev->irq_lock, flags);
} else {
REG_WRITE(0x80000101, RKNPU_OFFSET_CLR_ALL_RW_AMOUNT);
REG_WRITE(0x00000101, RKNPU_OFFSET_CLR_ALL_RW_AMOUNT);
spin_lock(&rknpu_dev->lock);
REG_WRITE(0x80000101, config->amount_top->offset_clr_all);
REG_WRITE(0x00000101, config->amount_top->offset_clr_all);
if (config->amount_core) {
REG_WRITE(0x80000101,
config->amount_core->offset_clr_all);
REG_WRITE(0x00000101,
config->amount_core->offset_clr_all);
}
spin_unlock(&rknpu_dev->lock);
}
spin_unlock(&rknpu_dev->lock);
return 0;
}
@ -884,23 +991,42 @@ int rknpu_get_rw_amount(struct rknpu_device *rknpu_dev, uint32_t *dt_wr,
uint32_t *dt_rd, uint32_t *wd_rd)
{
void __iomem *rknpu_core_base = rknpu_dev->base[0];
int amount_scale = rknpu_dev->config->pc_data_amount_scale;
const struct rknpu_config *config = rknpu_dev->config;
int amount_scale = config->pc_data_amount_scale;
if (!rknpu_dev->config->bw_enable) {
if (config->amount_top == NULL) {
LOG_WARN("Get rw_amount is not supported on this device!\n");
return 0;
}
spin_lock(&rknpu_dev->lock);
if (dt_wr != NULL)
*dt_wr = REG_READ(RKNPU_OFFSET_DT_WR_AMOUNT) * amount_scale;
if (dt_wr != NULL) {
*dt_wr = REG_READ(config->amount_top->offset_dt_wr) *
amount_scale;
if (config->amount_core) {
*dt_wr += REG_READ(config->amount_core->offset_dt_wr) *
amount_scale;
}
}
if (dt_rd != NULL)
*dt_rd = REG_READ(RKNPU_OFFSET_DT_RD_AMOUNT) * amount_scale;
if (dt_rd != NULL) {
*dt_rd = REG_READ(config->amount_top->offset_dt_rd) *
amount_scale;
if (config->amount_core) {
*dt_rd += REG_READ(config->amount_core->offset_dt_rd) *
amount_scale;
}
}
if (wd_rd != NULL)
*wd_rd = REG_READ(RKNPU_OFFSET_WT_RD_AMOUNT) * amount_scale;
if (wd_rd != NULL) {
*wd_rd = REG_READ(config->amount_top->offset_wt_rd) *
amount_scale;
if (config->amount_core) {
*wd_rd += REG_READ(config->amount_core->offset_wt_rd) *
amount_scale;
}
}
spin_unlock(&rknpu_dev->lock);
@ -909,12 +1035,13 @@ int rknpu_get_rw_amount(struct rknpu_device *rknpu_dev, uint32_t *dt_wr,
int rknpu_get_total_rw_amount(struct rknpu_device *rknpu_dev, uint32_t *amount)
{
const struct rknpu_config *config = rknpu_dev->config;
uint32_t dt_wr = 0;
uint32_t dt_rd = 0;
uint32_t wd_rd = 0;
int ret = -EINVAL;
if (!rknpu_dev->config->bw_enable) {
if (config->amount_top == NULL) {
LOG_WARN(
"Get total_rw_amount is not supported on this device!\n");
return 0;

127
target/linux/rockchip/rk35xx/files/drivers/rknpu/rknpu_mem.c
View File

@ -15,7 +15,10 @@
#include "rknpu_ioctl.h"
#include "rknpu_mem.h"
int rknpu_mem_create_ioctl(struct rknpu_device *rknpu_dev, unsigned long data)
#ifdef CONFIG_ROCKCHIP_RKNPU_DMA_HEAP
int rknpu_mem_create_ioctl(struct rknpu_device *rknpu_dev, struct file *file,
unsigned int cmd, unsigned long data)
{
struct rknpu_mem_create args;
int ret = -EINVAL;
@ -27,16 +30,23 @@ int rknpu_mem_create_ioctl(struct rknpu_device *rknpu_dev, unsigned long data)
struct page **pages;
struct page *page;
struct rknpu_mem_object *rknpu_obj = NULL;
struct rknpu_session *session = NULL;
int i, fd;
unsigned int length, page_count;
unsigned int in_size = _IOC_SIZE(cmd);
unsigned int k_size = sizeof(struct rknpu_mem_create);
char *k_data = (char *)&args;
if (unlikely(copy_from_user(&args, (struct rknpu_mem_create *)data,
sizeof(struct rknpu_mem_create)))) {
in_size))) {
LOG_ERROR("%s: copy_from_user failed\n", __func__);
ret = -EFAULT;
return ret;
}
if (k_size > in_size)
memset(k_data + in_size, 0, k_size - in_size);
if (args.flags & RKNPU_MEM_NON_CONTIGUOUS) {
LOG_ERROR("%s: malloc iommu memory unsupported in current!\n",
__func__);
@ -65,6 +75,8 @@ int rknpu_mem_create_ioctl(struct rknpu_device *rknpu_dev, unsigned long data)
O_CLOEXEC | O_RDWR, 0x0,
dev_name(rknpu_dev->dev));
if (IS_ERR(dmabuf)) {
LOG_ERROR("dmabuf alloc failed, args.size = %llu\n",
args.size);
ret = PTR_ERR(dmabuf);
goto err_free_obj;
}
@ -74,6 +86,7 @@ int rknpu_mem_create_ioctl(struct rknpu_device *rknpu_dev, unsigned long data)
fd = dma_buf_fd(dmabuf, O_CLOEXEC | O_RDWR);
if (fd < 0) {
LOG_ERROR("dmabuf fd get failed\n");
ret = -EFAULT;
goto err_free_dma_buf;
}
@ -81,12 +94,14 @@ int rknpu_mem_create_ioctl(struct rknpu_device *rknpu_dev, unsigned long data)
attachment = dma_buf_attach(dmabuf, rknpu_dev->dev);
if (IS_ERR(attachment)) {
LOG_ERROR("dma_buf_attach failed\n");
ret = PTR_ERR(attachment);
goto err_free_dma_buf;
}
table = dma_buf_map_attachment(attachment, DMA_BIDIRECTIONAL);
if (IS_ERR(table)) {
LOG_ERROR("dma_buf_attach failed\n");
dma_buf_detach(dmabuf, attachment);
ret = PTR_ERR(table);
goto err_free_dma_buf;
@ -100,20 +115,27 @@ int rknpu_mem_create_ioctl(struct rknpu_device *rknpu_dev, unsigned long data)
__LINE__, &phys, length);
}
page_count = length >> PAGE_SHIFT;
pages = kmalloc_array(page_count, sizeof(struct page), GFP_KERNEL);
if (!pages) {
ret = -ENOMEM;
goto err_detach_dma_buf;
}
if (args.flags & RKNPU_MEM_KERNEL_MAPPING) {
page_count = length >> PAGE_SHIFT;
pages = vmalloc(page_count * sizeof(struct page));
if (!pages) {
LOG_ERROR("alloc pages failed\n");
ret = -ENOMEM;
goto err_detach_dma_buf;
}
for (i = 0; i < page_count; i++)
pages[i] = &page[i];
for (i = 0; i < page_count; i++)
pages[i] = &page[i];
rknpu_obj->kv_addr = vmap(pages, page_count, VM_MAP, PAGE_KERNEL);
if (!rknpu_obj->kv_addr) {
ret = -ENOMEM;
goto err_free_pages;
rknpu_obj->kv_addr =
vmap(pages, page_count, VM_MAP, PAGE_KERNEL);
if (!rknpu_obj->kv_addr) {
LOG_ERROR("vmap pages addr failed\n");
ret = -ENOMEM;
goto err_free_pages;
}
vfree(pages);
pages = NULL;
}
rknpu_obj->size = PAGE_ALIGN(args.size);
@ -131,16 +153,27 @@ int rknpu_mem_create_ioctl(struct rknpu_device *rknpu_dev, unsigned long data)
(__u64)rknpu_obj->dma_addr);
if (unlikely(copy_to_user((struct rknpu_mem_create *)data, &args,
sizeof(struct rknpu_mem_create)))) {
in_size))) {
LOG_ERROR("%s: copy_to_user failed\n", __func__);
ret = -EFAULT;
goto err_unmap_kv_addr;
}
kfree(pages);
dma_buf_unmap_attachment(attachment, table, DMA_BIDIRECTIONAL);
dma_buf_detach(dmabuf, attachment);
spin_lock(&rknpu_dev->lock);
session = file->private_data;
if (!session) {
spin_unlock(&rknpu_dev->lock);
ret = -EFAULT;
goto err_unmap_kv_addr;
}
list_add_tail(&rknpu_obj->head, &session->list);
spin_unlock(&rknpu_dev->lock);
return 0;
err_unmap_kv_addr:
@ -148,7 +181,8 @@ err_unmap_kv_addr:
rknpu_obj->kv_addr = NULL;
err_free_pages:
kfree(pages);
vfree(pages);
pages = NULL;
err_detach_dma_buf:
dma_buf_unmap_attachment(attachment, table, DMA_BIDIRECTIONAL);
@ -166,11 +200,12 @@ err_free_obj:
return ret;
}
int rknpu_mem_destroy_ioctl(struct rknpu_device *rknpu_dev, unsigned long data)
int rknpu_mem_destroy_ioctl(struct rknpu_device *rknpu_dev, struct file *file,
unsigned long data)
{
struct rknpu_mem_object *rknpu_obj = NULL;
struct rknpu_mem_object *rknpu_obj, *entry, *q;
struct rknpu_session *session = NULL;
struct rknpu_mem_destroy args;
struct dma_buf *dmabuf;
int ret = -EFAULT;
if (unlikely(copy_from_user(&args, (struct rknpu_mem_destroy *)data,
@ -188,19 +223,35 @@ int rknpu_mem_destroy_ioctl(struct rknpu_device *rknpu_dev, unsigned long data)
}
rknpu_obj = (struct rknpu_mem_object *)(uintptr_t)args.obj_addr;
dmabuf = rknpu_obj->dmabuf;
LOG_DEBUG(
"free args.handle: %d, rknpu_obj: %#llx, rknpu_obj->dma_addr: %#llx\n",
args.handle, (__u64)(uintptr_t)rknpu_obj,
(__u64)rknpu_obj->dma_addr);
vunmap(rknpu_obj->kv_addr);
rknpu_obj->kv_addr = NULL;
spin_lock(&rknpu_dev->lock);
session = file->private_data;
if (!session) {
spin_unlock(&rknpu_dev->lock);
ret = -EFAULT;
return ret;
}
list_for_each_entry_safe(entry, q, &session->list, head) {
if (entry == rknpu_obj) {
list_del(&entry->head);
break;
}
}
spin_unlock(&rknpu_dev->lock);
if (!rknpu_obj->owner)
dma_buf_put(dmabuf);
if (rknpu_obj == entry) {
vunmap(rknpu_obj->kv_addr);
rknpu_obj->kv_addr = NULL;
kfree(rknpu_obj);
if (!rknpu_obj->owner)
dma_buf_put(rknpu_obj->dmabuf);
kfree(rknpu_obj);
}
return 0;
}
@ -209,11 +260,9 @@ int rknpu_mem_destroy_ioctl(struct rknpu_device *rknpu_dev, unsigned long data)
* begin cpu access => for_cpu = true
* end cpu access => for_cpu = false
*/
static void __maybe_unused rknpu_dma_buf_sync(struct rknpu_device *rknpu_dev,
struct rknpu_mem_object *rknpu_obj,
u32 offset, u32 length,
enum dma_data_direction dir,
bool for_cpu)
static void __maybe_unused rknpu_dma_buf_sync(
struct rknpu_device *rknpu_dev, struct rknpu_mem_object *rknpu_obj,
u32 offset, u32 length, enum dma_data_direction dir, bool for_cpu)
{
struct device *dev = rknpu_dev->dev;
struct sg_table *sgt = rknpu_obj->sgt;
@ -256,7 +305,9 @@ int rknpu_mem_sync_ioctl(struct rknpu_device *rknpu_dev, unsigned long data)
{
struct rknpu_mem_object *rknpu_obj = NULL;
struct rknpu_mem_sync args;
#ifdef CONFIG_DMABUF_PARTIAL
struct dma_buf *dmabuf;
#endif
int ret = -EFAULT;
if (unlikely(copy_from_user(&args, (struct rknpu_mem_sync *)data,
@ -274,20 +325,18 @@ int rknpu_mem_sync_ioctl(struct rknpu_device *rknpu_dev, unsigned long data)
}
rknpu_obj = (struct rknpu_mem_object *)(uintptr_t)args.obj_addr;
dmabuf = rknpu_obj->dmabuf;
#ifndef CONFIG_DMABUF_PARTIAL
if (args.flags & RKNPU_MEM_SYNC_TO_DEVICE) {
rknpu_dma_buf_sync(rknpu_dev, rknpu_obj,
args.offset, args.size, DMA_TO_DEVICE,
false);
rknpu_dma_buf_sync(rknpu_dev, rknpu_obj, args.offset, args.size,
DMA_TO_DEVICE, false);
}
if (args.flags & RKNPU_MEM_SYNC_FROM_DEVICE) {
rknpu_dma_buf_sync(rknpu_dev, rknpu_obj,
args.offset, args.size, DMA_FROM_DEVICE,
true);
rknpu_dma_buf_sync(rknpu_dev, rknpu_obj, args.offset, args.size,
DMA_FROM_DEVICE, true);
}
#else
dmabuf = rknpu_obj->dmabuf;
if (args.flags & RKNPU_MEM_SYNC_TO_DEVICE) {
dmabuf->ops->end_cpu_access_partial(dmabuf, DMA_TO_DEVICE,
args.offset, args.size);
@ -300,3 +349,5 @@ int rknpu_mem_sync_ioctl(struct rknpu_device *rknpu_dev, unsigned long data)
return 0;
}
#endif

51
target/linux/rockchip/rk35xx/files/drivers/rknpu/rknpu_mm.c
View File

@ -236,54 +236,3 @@ int rknpu_mm_dump(struct seq_file *m, void *data)
return 0;
}
dma_addr_t rknpu_iommu_dma_alloc_iova(struct iommu_domain *domain, size_t size,
u64 dma_limit, struct device *dev)
{
struct rknpu_iommu_dma_cookie *cookie = domain->iova_cookie;
struct iova_domain *iovad = &cookie->iovad;
unsigned long shift, iova_len, iova = 0;
#if (KERNEL_VERSION(5, 4, 0) > LINUX_VERSION_CODE)
dma_addr_t limit;
#endif
shift = iova_shift(iovad);
iova_len = size >> shift;
/*
* Freeing non-power-of-two-sized allocations back into the IOVA caches
* will come back to bite us badly, so we have to waste a bit of space
* rounding up anything cacheable to make sure that can't happen. The
* order of the unadjusted size will still match upon freeing.
*/
if (iova_len < (1 << (IOVA_RANGE_CACHE_MAX_SIZE - 1)))
iova_len = roundup_pow_of_two(iova_len);
#if (KERNEL_VERSION(5, 10, 0) <= LINUX_VERSION_CODE)
dma_limit = min_not_zero(dma_limit, dev->bus_dma_limit);
#else
if (dev->bus_dma_mask)
dma_limit &= dev->bus_dma_mask;
#endif
if (domain->geometry.force_aperture)
dma_limit =
min_t(u64, dma_limit, domain->geometry.aperture_end);
#if (KERNEL_VERSION(5, 4, 0) <= LINUX_VERSION_CODE)
iova = alloc_iova_fast(iovad, iova_len, dma_limit >> shift, true);
#else
limit = min_t(dma_addr_t, dma_limit >> shift, iovad->end_pfn);
iova = alloc_iova_fast(iovad, iova_len, limit, true);
#endif
return (dma_addr_t)iova << shift;
}
void rknpu_iommu_dma_free_iova(struct rknpu_iommu_dma_cookie *cookie,
dma_addr_t iova, size_t size)
{
struct iova_domain *iovad = &cookie->iovad;
free_iova_fast(iovad, iova_pfn(iovad, iova), size >> iova_shift(iovad));
}

64
target/linux/rockchip/rk35xx/files/drivers/rknpu/rknpu_reset.c
View File

@ -28,27 +28,34 @@ static inline struct reset_control *rknpu_reset_control_get(struct device *dev,
int rknpu_reset_get(struct rknpu_device *rknpu_dev)
{
#ifndef FPGA_PLATFORM
struct reset_control *srst_a = NULL;
struct reset_control *srst_h = NULL;
int i = 0;
int num_srsts = 0;
for (i = 0; i < rknpu_dev->config->num_resets; i++) {
srst_a = rknpu_reset_control_get(
rknpu_dev->dev,
rknpu_dev->config->resets[i].srst_a_name);
if (IS_ERR(srst_a))
return PTR_ERR(srst_a);
rknpu_dev->srst_a[i] = srst_a;
srst_h = rknpu_reset_control_get(
rknpu_dev->dev,
rknpu_dev->config->resets[i].srst_h_name);
if (IS_ERR(srst_h))
return PTR_ERR(srst_h);
rknpu_dev->srst_h[i] = srst_h;
num_srsts = of_count_phandle_with_args(rknpu_dev->dev->of_node,
"resets", "#reset-cells");
if (num_srsts <= 0) {
LOG_DEV_ERROR(rknpu_dev->dev,
"failed to get rknpu resets from dtb\n");
return num_srsts;
}
rknpu_dev->srsts = devm_kcalloc(rknpu_dev->dev, num_srsts,
sizeof(*rknpu_dev->srsts), GFP_KERNEL);
if (!rknpu_dev->srsts)
return -ENOMEM;
for (i = 0; i < num_srsts; ++i) {
rknpu_dev->srsts[i] = devm_reset_control_get_exclusive_by_index(
rknpu_dev->dev, i);
if (IS_ERR(rknpu_dev->srsts[i])) {
rknpu_dev->num_srsts = i;
return PTR_ERR(rknpu_dev->srsts[i]);
}
}
rknpu_dev->num_srsts = num_srsts;
return num_srsts;
#endif
return 0;
@ -93,7 +100,7 @@ int rknpu_soft_reset(struct rknpu_device *rknpu_dev)
#ifndef FPGA_PLATFORM
struct iommu_domain *domain = NULL;
struct rknpu_subcore_data *subcore_data = NULL;
int ret = -EINVAL, i = 0;
int ret = 0, i = 0;
if (rknpu_dev->bypass_soft_reset) {
LOG_WARN("bypass soft reset\n");
@ -112,17 +119,17 @@ int rknpu_soft_reset(struct rknpu_device *rknpu_dev)
wake_up(&subcore_data->job_done_wq);
}
LOG_INFO("soft reset\n");
LOG_INFO("soft reset, num: %d\n", rknpu_dev->num_srsts);
for (i = 0; i < rknpu_dev->config->num_resets; i++) {
ret = rknpu_reset_assert(rknpu_dev->srst_a[i]);
ret |= rknpu_reset_assert(rknpu_dev->srst_h[i]);
for (i = 0; i < rknpu_dev->num_srsts; ++i)
ret |= rknpu_reset_assert(rknpu_dev->srsts[i]);
udelay(10);
udelay(10);
ret |= rknpu_reset_deassert(rknpu_dev->srst_a[i]);
ret |= rknpu_reset_deassert(rknpu_dev->srst_h[i]);
}
for (i = 0; i < rknpu_dev->num_srsts; ++i)
ret |= rknpu_reset_deassert(rknpu_dev->srsts[i]);
udelay(10);
if (ret) {
LOG_DEV_ERROR(rknpu_dev->dev,
@ -141,6 +148,9 @@ int rknpu_soft_reset(struct rknpu_device *rknpu_dev)
rknpu_dev->soft_reseting = false;
if (rknpu_dev->config->state_init != NULL)
rknpu_dev->config->state_init(rknpu_dev);
mutex_unlock(&rknpu_dev->reset_lock);
#endif

24
target/linux/rockchip/rk35xx/patches-5.10/1027-export-iommu_setup_dma_ops.patch Normal file
View File

@ -0,0 +1,24 @@
From 2f679264ca2dc5e9b34e4690740490b52f1f21a8 Mon Sep 17 00:00:00 2001
From: jjm2473 <jjm2473@gmail.com>
Date: Wed, 12 Feb 2025 12:00:55 +0800
Subject: [PATCH] dma-iommu: export iommu_setup_dma_ops
---
drivers/iommu/dma-iommu.c | 1 +
1 file changed, 1 insertion(+)
diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c
index b6452257d..10ed1b8ff 100644
--- a/drivers/iommu/dma-iommu.c
+++ b/drivers/iommu/dma-iommu.c
@@ -1243,6 +1243,7 @@ void iommu_setup_dma_ops(struct device *dev, u64 dma_base, u64 size)
pr_warn("Failed to set up IOMMU for device %s; retaining platform DMA ops\n",
dev_name(dev));
}
+EXPORT_SYMBOL_GPL(iommu_setup_dma_ops);
static struct iommu_dma_msi_page *iommu_dma_get_msi_page(struct device *dev,
phys_addr_t msi_addr, struct iommu_domain *domain)
--
2.46.0

206
target/linux/rockchip/rk35xx/patches-5.10/1028-drivers-rknpu-backport.patch Normal file
View File

@ -0,0 +1,206 @@
From e16a5515f68cced6b8f54880bc3bd22702a0528a Mon Sep 17 00:00:00 2001
From: jjm2473 <jjm2473@gmail.com>
Date: Tue, 11 Feb 2025 13:51:29 +0800
Subject: [PATCH] drivers/rknpu: backport to 5.10
---
drivers/rknpu/rknpu_devfreq.c | 8 ++++----
drivers/rknpu/rknpu_drv.c | 10 ++++++++++
drivers/rknpu/rknpu_gem.c | 23 +++++++++++++++++++++++
3 files changed, 37 insertions(+), 4 deletions(-)
diff --git a/drivers/rknpu/rknpu_devfreq.c b/drivers/rknpu/rknpu_devfreq.c
index 2668d2c46..ea717dde2 100644
--- a/drivers/rknpu/rknpu_devfreq.c
+++ b/drivers/rknpu/rknpu_devfreq.c
@@ -19,7 +19,7 @@ static int npu_devfreq_target(struct device *dev, unsigned long *freq,
u32 flags);
static struct monitor_dev_profile npu_mdevp = {
- .type = MONITOR_TYPE_DEV,
+ .type = MONITOR_TPYE_DEV,
.low_temp_adjust = rockchip_monitor_dev_low_temp_adjust,
.high_temp_adjust = rockchip_monitor_dev_high_temp_adjust,
#if KERNEL_VERSION(6, 1, 0) <= LINUX_VERSION_CODE
@@ -395,7 +395,7 @@ int rknpu_devfreq_init(struct rknpu_device *rknpu_dev)
err_remove_governor:
devfreq_remove_governor(&devfreq_rknpu_ondemand);
err_uinit_table:
-#if KERNEL_VERSION(5, 10, 198) <= LINUX_VERSION_CODE
+#if KERNEL_VERSION(6, 1, 0) <= LINUX_VERSION_CODE
rockchip_uninit_opp_table(dev, info);
#else
dev_pm_opp_of_remove_table(dev);
@@ -733,7 +733,7 @@ int rknpu_devfreq_init(struct rknpu_device *rknpu_dev)
err_remove_governor:
devfreq_remove_governor(&devfreq_rknpu_ondemand);
err_remove_table:
-#if KERNEL_VERSION(5, 10, 198) <= LINUX_VERSION_CODE
+#if KERNEL_VERSION(6, 1, 0) <= LINUX_VERSION_CODE
rockchip_uninit_opp_table(dev, &rknpu_dev->opp_info);
#else
dev_pm_opp_of_remove_table(dev);
@@ -798,7 +798,7 @@ void rknpu_devfreq_remove(struct rknpu_device *rknpu_dev)
}
if (rknpu_dev->devfreq)
devfreq_remove_governor(&devfreq_rknpu_ondemand);
-#if KERNEL_VERSION(5, 10, 198) <= LINUX_VERSION_CODE
+#if KERNEL_VERSION(6, 1, 0) <= LINUX_VERSION_CODE
rockchip_uninit_opp_table(rknpu_dev->dev, &rknpu_dev->opp_info);
#else
dev_pm_opp_of_remove_table(rknpu_dev->dev);
diff --git a/drivers/rknpu/rknpu_drv.c b/drivers/rknpu/rknpu_drv.c
index e8afa9778..25fa35aec 100644
--- a/drivers/rknpu/rknpu_drv.c
+++ b/drivers/rknpu/rknpu_drv.c
@@ -108,6 +108,7 @@ static const struct rknpu_amount_data rknpu_core_amount = {
.offset_wt_rd = 0x243c,
};
+#ifdef CONFIG_CPU_RK3576
static void rk3576_state_init(struct rknpu_device *rknpu_dev)
{
void __iomem *rknpu_core_base = rknpu_dev->base[0];
@@ -158,6 +159,7 @@ static int rk3576_cache_sgt_init(struct rknpu_device *rknpu_dev)
return ret;
}
+#endif
static const struct rknpu_config rk356x_rknpu_config = {
.bw_priority_addr = 0xfe180008,
@@ -243,6 +245,7 @@ static const struct rknpu_config rv1106_rknpu_config = {
.cache_sgt_init = NULL,
};
+#ifdef CONFIG_CPU_RK3562
static const struct rknpu_config rk3562_rknpu_config = {
.bw_priority_addr = 0x0,
.bw_priority_length = 0x0,
@@ -263,7 +266,9 @@ static const struct rknpu_config rk3562_rknpu_config = {
.state_init = NULL,
.cache_sgt_init = NULL,
};
+#endif
+#ifdef CONFIG_CPU_RK3576
static const struct rknpu_config rk3576_rknpu_config = {
.bw_priority_addr = 0x0,
.bw_priority_length = 0x0,
@@ -284,6 +289,7 @@ static const struct rknpu_config rk3576_rknpu_config = {
.state_init = rk3576_state_init,
.cache_sgt_init = rk3576_cache_sgt_init,
};
+#endif
/* driver probe and init */
static const struct of_device_id rknpu_of_match[] = {
@@ -303,14 +309,18 @@ static const struct of_device_id rknpu_of_match[] = {
.compatible = "rockchip,rv1106-rknpu",
.data = &rv1106_rknpu_config,
},
+#ifdef CONFIG_CPU_RK3562
{
.compatible = "rockchip,rk3562-rknpu",
.data = &rk3562_rknpu_config,
},
+#endif
+#ifdef CONFIG_CPU_RK3576
{
.compatible = "rockchip,rk3576-rknpu",
.data = &rk3576_rknpu_config,
},
+#endif
{},
};
diff --git a/drivers/rknpu/rknpu_gem.c b/drivers/rknpu/rknpu_gem.c
index a5c5354ed..14fdf643b 100644
--- a/drivers/rknpu/rknpu_gem.c
+++ b/drivers/rknpu/rknpu_gem.c
@@ -28,6 +28,12 @@
#include "rknpu_gem.h"
#include "rknpu_iommu.h"
+#if IS_ENABLED(CONFIG_NO_GKI) && (IS_ENABLED(CONFIG_ROCKCHIP_RKNPU_SRAM) || defined(CONFIG_CPU_RK3562) || defined(CONFIG_CPU_RK3576))
+#define RKNPU_CACHE_SYNC 1
+#else
+#define RKNPU_CACHE_SYNC 0
+#endif
+
#define RKNPU_GEM_ALLOC_FROM_PAGES 1
#if RKNPU_GEM_ALLOC_FROM_PAGES
@@ -995,7 +1001,11 @@ static int rknpu_gem_mmap_pages(struct rknpu_gem_object *rknpu_obj,
struct drm_device *drm = rknpu_obj->base.dev;
int ret = -EINVAL;
+#if KERNEL_VERSION(6, 3, 0) > LINUX_VERSION_CODE
+ vma->vm_flags |= VM_MIXEDMAP;
+#else
vm_flags_set(vma, VM_MIXEDMAP);
+#endif
ret = __vm_map_pages(vma, rknpu_obj->pages, rknpu_obj->num_pages,
vma->vm_pgoff);
@@ -1088,7 +1098,11 @@ static int rknpu_gem_mmap_cache(struct rknpu_gem_object *rknpu_obj,
return -EINVAL;
}
+#if KERNEL_VERSION(6, 3, 0) > LINUX_VERSION_CODE
+ vma->vm_flags |= VM_MIXEDMAP;
+#else
vm_flags_set(vma, VM_MIXEDMAP);
+#endif
vm_size = vma->vm_end - vma->vm_start;
@@ -1144,8 +1158,13 @@ static int rknpu_gem_mmap_buffer(struct rknpu_gem_object *rknpu_obj,
* vm_pgoff (used as a fake buffer offset by DRM) to 0 as we want to map
* the whole buffer.
*/
+#if KERNEL_VERSION(6, 3, 0) > LINUX_VERSION_CODE
+ vma->vm_flags |= VM_DONTCOPY | VM_DONTEXPAND | VM_DONTDUMP | VM_IO;
+ vma->vm_flags &= ~VM_PFNMAP;
+#else
vm_flags_set(vma, VM_DONTCOPY | VM_DONTEXPAND | VM_DONTDUMP | VM_IO);
vm_flags_clear(vma, VM_PFNMAP);
+#endif
vma->vm_pgoff = 0;
vm_size = vma->vm_end - vma->vm_start;
@@ -1530,6 +1549,7 @@ int rknpu_gem_prime_mmap(struct drm_gem_object *obj, struct vm_area_struct *vma)
return rknpu_gem_mmap_obj(obj, vma);
}
+#if RKNPU_CACHE_SYNC
static int rknpu_cache_sync_with_sg(struct rknpu_device *rknpu_dev,
struct rknpu_gem_object *rknpu_obj,
unsigned long *length,
@@ -1663,6 +1683,7 @@ static int rknpu_cache_sync(struct rknpu_gem_object *rknpu_obj,
return 0;
}
+#endif
int rknpu_gem_sync_ioctl(struct drm_device *dev, void *data,
struct drm_file *file_priv)
@@ -1708,6 +1729,7 @@ int rknpu_gem_sync_ioctl(struct drm_device *dev, void *data,
length = args->size;
offset = args->offset;
+#if RKNPU_CACHE_SYNC
if (IS_ENABLED(CONFIG_NO_GKI) &&
IS_ENABLED(CONFIG_ROCKCHIP_RKNPU_SRAM) &&
rknpu_obj->sram_size > 0) {
@@ -1718,6 +1740,7 @@ int rknpu_gem_sync_ioctl(struct drm_device *dev, void *data,
rknpu_cache_sync(rknpu_obj, &length, &offset,
RKNPU_CACHE_NBUF, args->flags);
}
+#endif
for_each_sg(rknpu_obj->sgt->sgl, sg, rknpu_obj->sgt->nents, i) {
if (length == 0)
--
2.46.0